From 0fa67c1d686c1c25b467906307cafefa885c4a80 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 27 Dec 2020 08:55:51 +0100
Subject: [PATCH 001/860] [generic] Add support for VHX Embeds(#27546)

---
 youtube_dl/extractor/generic.py | 22 ++++++++++++++++++++--
 youtube_dl/extractor/vimeo.py   |  7 +++++++
 2 files changed, 27 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 6e46b2c0e..14c27c6da 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -67,7 +67,10 @@ from .tube8 import Tube8IE
 from .mofosex import MofosexEmbedIE
 from .spankwire import SpankwireIE
 from .youporn import YouPornIE
-from .vimeo import VimeoIE
+from .vimeo import (
+    VimeoIE,
+    VHXEmbedIE,
+)
 from .dailymotion import DailymotionIE
 from .dailymail import DailyMailIE
 from .onionstudios import OnionStudiosIE
@@ -2193,7 +2196,18 @@ class GenericIE(InfoExtractor):
         #     'params': {
         #         'force_generic_extractor': True,
         #     },
-        # }
+        # },
+        {
+            # VHX Embed
+            'url': 'https://demo.vhx.tv/category-c/videos/file-example-mp4-480-1-5mg-copy',
+            'info_dict': {
+                'id': '858208',
+                'ext': 'mp4',
+                'title': 'Untitled',
+                'uploader_id': 'user80538407',
+                'uploader': 'OTT Videos',
+            },
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -2571,6 +2585,10 @@ class GenericIE(InfoExtractor):
         if vimeo_urls:
             return self.playlist_from_matches(vimeo_urls, video_id, video_title, ie=VimeoIE.ie_key())
 
+        vhx_url = VHXEmbedIE._extract_url(webpage)
+        if vhx_url:
+            return self.url_result(vhx_url, VHXEmbedIE.ie_key())
+
         vid_me_embed_url = self._search_regex(
             r'src=[\'"](https?://vid\.me/[^\'"]+)[\'"]',
             webpage, 'vid.me embed', default=None)
diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 4c55946f1..15cd06268 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -1119,6 +1119,12 @@ class VHXEmbedIE(VimeoBaseInfoExtractor):
     IE_NAME = 'vhx:embed'
     _VALID_URL = r'https?://embed\.vhx\.tv/videos/(?P<id>\d+)'
 
+    @staticmethod
+    def _extract_url(webpage):
+        mobj = re.search(
+            r'<iframe[^>]+src="(https?://embed\.vhx\.tv/videos/\d+[^"]*)"', webpage)
+        return unescapeHTML(mobj.group(1)) if mobj else None
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
@@ -1127,5 +1133,6 @@ class VHXEmbedIE(VimeoBaseInfoExtractor):
             'ott data'), video_id, js_to_json)['config_url']
         config = self._download_json(config_url, video_id)
         info = self._parse_config(config, video_id)
+        info['id'] = video_id
         self._vimeo_sort_formats(info['formats'])
         return info

From aed617e311b95d771e10f48cc8dcde25d7816224 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 27 Dec 2020 08:58:34 +0100
Subject: [PATCH 002/860] [amcnetworks] improve auth only video
 detection(closes #27548)

---
 youtube_dl/extractor/amcnetworks.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/amcnetworks.py b/youtube_dl/extractor/amcnetworks.py
index 12b6de0bf..b8027bbca 100644
--- a/youtube_dl/extractor/amcnetworks.py
+++ b/youtube_dl/extractor/amcnetworks.py
@@ -80,7 +80,8 @@ class AMCNetworksIE(ThePlatformIE):
         title = theplatform_metadata['title']
         rating = try_get(
             theplatform_metadata, lambda x: x['ratings'][0]['rating'])
-        if properties.get('videoCategory') == 'TVE-Auth':
+        video_category = properties.get('videoCategory')
+        if video_category and video_category.endswith('-Auth'):
             resource = self._get_mvpd_resource(
                 requestor_id, title, video_id, rating)
             query['auth'] = self._extract_mvpd_auth(

From e4749965412edf2c6d3938d4b8f1d3dbab61b0b5 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 27 Dec 2020 21:15:09 +0700
Subject: [PATCH 003/860] [youtube] Update invidious.snopyta.org (#22667)

Co-authored-by: sofutru <54445344+sofutru@users.noreply.github.com>
---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 289d9bab1..5ef58d730 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -321,7 +321,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             # Invidious instances taken from https://github.com/omarroth/invidious/wiki/Invidious-Instances
                             (?:(?:www|dev)\.)?invidio\.us/|
                             (?:(?:www|no)\.)?invidiou\.sh/|
-                            (?:(?:www|fi|de)\.)?invidious\.snopyta\.org/|
+                            (?:(?:www|fi)\.)?invidious\.snopyta\.org/|
                             (?:www\.)?invidious\.kabi\.tk/|
                             (?:www\.)?invidious\.13ad\.de/|
                             (?:www\.)?invidious\.mastodon\.host/|

From f86b299d0ecdba3462d67247cf70cee3a2809a1f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 27 Dec 2020 16:20:49 +0100
Subject: [PATCH 004/860] [telecinco] fix extraction

---
 youtube_dl/extractor/telecinco.py | 77 ++++++++-----------------------
 1 file changed, 20 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/telecinco.py b/youtube_dl/extractor/telecinco.py
index 9ba3da341..eecd6a5c9 100644
--- a/youtube_dl/extractor/telecinco.py
+++ b/youtube_dl/extractor/telecinco.py
@@ -5,14 +5,11 @@ import json
 import re
 
 from .common import InfoExtractor
-from .ooyala import OoyalaIE
 from ..utils import (
     clean_html,
-    determine_ext,
     int_or_none,
     str_or_none,
     try_get,
-    urljoin,
 )
 
 
@@ -28,7 +25,7 @@ class TelecincoIE(InfoExtractor):
             'description': 'md5:716caf5601e25c3c5ab6605b1ae71529',
         },
         'playlist': [{
-            'md5': 'adb28c37238b675dad0f042292f209a7',
+            'md5': '7ee56d665cfd241c0e6d80fd175068b0',
             'info_dict': {
                 'id': 'JEA5ijCnF6p5W08A1rNKn7',
                 'ext': 'mp4',
@@ -38,7 +35,7 @@ class TelecincoIE(InfoExtractor):
         }]
     }, {
         'url': 'http://www.cuatro.com/deportes/futbol/barcelona/Leo_Messi-Champions-Roma_2_2052780128.html',
-        'md5': '9468140ebc300fbb8b9d65dc6e5c4b43',
+        'md5': 'c86fe0d99e3bdb46b7950d38bf6ef12a',
         'info_dict': {
             'id': 'jn24Od1zGLG4XUZcnUnZB6',
             'ext': 'mp4',
@@ -48,7 +45,7 @@ class TelecincoIE(InfoExtractor):
         },
     }, {
         'url': 'http://www.mediaset.es/12meses/campanas/doylacara/conlatratanohaytrato/Ayudame-dar-cara-trata-trato_2_1986630220.html',
-        'md5': 'ae2dc6b7b50b2392076a51c0f70e01f6',
+        'md5': 'eddb50291df704ce23c74821b995bcac',
         'info_dict': {
             'id': 'aywerkD2Sv1vGNqq9b85Q2',
             'ext': 'mp4',
@@ -90,58 +87,24 @@ class TelecincoIE(InfoExtractor):
 
     def _parse_content(self, content, url):
         video_id = content['dataMediaId']
-        if content.get('dataCmsId') == 'ooyala':
-            return self.url_result(
-                'ooyala:%s' % video_id, OoyalaIE.ie_key(), video_id)
-        config_url = urljoin(url, content['dataConfig'])
         config = self._download_json(
-            config_url, video_id, 'Downloading config JSON')
+            content['dataConfig'], video_id, 'Downloading config JSON')
         title = config['info']['title']
-
-        def mmc_url(mmc_type):
-            return re.sub(
-                r'/(?:flash|html5)\.json', '/%s.json' % mmc_type,
-                config['services']['mmc'])
-
-        duration = None
-        formats = []
-        for mmc_type in ('flash', 'html5'):
-            mmc = self._download_json(
-                mmc_url(mmc_type), video_id,
-                'Downloading %s mmc JSON' % mmc_type, fatal=False)
-            if not mmc:
-                continue
-            if not duration:
-                duration = int_or_none(mmc.get('duration'))
-            for location in mmc['locations']:
-                gat = self._proto_relative_url(location.get('gat'), 'http:')
-                gcp = location.get('gcp')
-                ogn = location.get('ogn')
-                if None in (gat, gcp, ogn):
-                    continue
-                token_data = {
-                    'gcp': gcp,
-                    'ogn': ogn,
-                    'sta': 0,
-                }
-                media = self._download_json(
-                    gat, video_id, data=json.dumps(token_data).encode('utf-8'),
-                    headers={
-                        'Content-Type': 'application/json;charset=utf-8',
-                        'Referer': url,
-                    }, fatal=False) or {}
-                stream = media.get('stream') or media.get('file')
-                if not stream:
-                    continue
-                ext = determine_ext(stream)
-                if ext == 'f4m':
-                    formats.extend(self._extract_f4m_formats(
-                        stream + '&hdcore=3.2.0&plugin=aasp-3.2.0.77.18',
-                        video_id, f4m_id='hds', fatal=False))
-                elif ext == 'm3u8':
-                    formats.extend(self._extract_m3u8_formats(
-                        stream, video_id, 'mp4', 'm3u8_native',
-                        m3u8_id='hls', fatal=False))
+        services = config['services']
+        caronte = self._download_json(services['caronte'], video_id)
+        stream = caronte['dls'][0]['stream']
+        headers = self.geo_verification_headers()
+        headers.update({
+            'Content-Type': 'application/json;charset=UTF-8',
+            'Origin': re.match(r'https?://[^/]+', url).group(0),
+        })
+        cdn = self._download_json(
+            caronte['cerbero'], video_id, data=json.dumps({
+                'bbx': caronte['bbx'],
+                'gbx': self._download_json(services['gbx'], video_id)['gbx'],
+            }).encode(), headers=headers)['tokens']['1']['cdn']
+        formats = self._extract_m3u8_formats(
+            stream + '?' + cdn, video_id, 'mp4', 'm3u8_native', m3u8_id='hls')
         self._sort_formats(formats)
 
         return {
@@ -149,7 +112,7 @@ class TelecincoIE(InfoExtractor):
             'title': title,
             'formats': formats,
             'thumbnail': content.get('dataPoster') or config.get('poster', {}).get('imageUrl'),
-            'duration': duration,
+            'duration': int_or_none(content.get('dataDuration')),
         }
 
     def _real_extract(self, url):

From 4c7a4dbc4d07786734324b267a41b74ee7099a85 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 27 Dec 2020 16:22:01 +0100
Subject: [PATCH 005/860] [mitele] fix free video extraction(#24624)(closes
 #25827)(closes #26757)

---
 youtube_dl/extractor/mitele.py | 20 ++++++--------------
 1 file changed, 6 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/mitele.py b/youtube_dl/extractor/mitele.py
index ad9da9612..b5937233b 100644
--- a/youtube_dl/extractor/mitele.py
+++ b/youtube_dl/extractor/mitele.py
@@ -1,15 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .common import InfoExtractor
+from .telecinco import TelecincoIE
 from ..utils import (
     int_or_none,
     parse_iso8601,
-    smuggle_url,
 )
 
 
-class MiTeleIE(InfoExtractor):
+class MiTeleIE(TelecincoIE):
     IE_DESC = 'mitele.es'
     _VALID_URL = r'https?://(?:www\.)?mitele\.es/(?:[^/]+/)+(?P<id>[^/]+)/player'
 
@@ -31,7 +30,6 @@ class MiTeleIE(InfoExtractor):
             'timestamp': 1471209401,
             'upload_date': '20160814',
         },
-        'add_ie': ['Ooyala'],
     }, {
         # no explicit title
         'url': 'http://www.mitele.es/programas-tv/cuarto-milenio/57b0de3dc915da14058b4876/player',
@@ -54,7 +52,6 @@ class MiTeleIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
-        'add_ie': ['Ooyala'],
     }, {
         'url': 'http://www.mitele.es/series-online/la-que-se-avecina/57aac5c1c915da951a8b45ed/player',
         'only_matching': True,
@@ -70,16 +67,11 @@ class MiTeleIE(InfoExtractor):
             r'window\.\$REACTBASE_STATE\.prePlayer_mtweb\s*=\s*({.+})',
             webpage, 'Pre Player'), display_id)['prePlayer']
         title = pre_player['title']
-        video = pre_player['video']
-        video_id = video['dataMediaId']
+        video_info = self._parse_content(pre_player['video'], url)
         content = pre_player.get('content') or {}
         info = content.get('info') or {}
 
-        return {
-            '_type': 'url_transparent',
-            # for some reason only HLS is supported
-            'url': smuggle_url('ooyala:' + video_id, {'supportedformats': 'm3u8,dash'}),
-            'id': video_id,
+        video_info.update({
             'title': title,
             'description': info.get('synopsis'),
             'series': content.get('title'),
@@ -87,7 +79,7 @@ class MiTeleIE(InfoExtractor):
             'episode': content.get('subtitle'),
             'episode_number': int_or_none(info.get('episode_number')),
             'duration': int_or_none(info.get('duration')),
-            'thumbnail': video.get('dataPoster'),
             'age_limit': int_or_none(info.get('rating')),
             'timestamp': parse_iso8601(pre_player.get('publishedTime')),
-        }
+        })
+        return video_info

From 6f2eaaf73daef3ac0995cd7b51c677b003c04218 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 27 Dec 2020 22:57:50 +0700
Subject: [PATCH 006/860] [teachable] Improve embed detection (closes #26923)

---
 youtube_dl/extractor/teachable.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/teachable.py b/youtube_dl/extractor/teachable.py
index 6f264bddc..2394f86d4 100644
--- a/youtube_dl/extractor/teachable.py
+++ b/youtube_dl/extractor/teachable.py
@@ -140,7 +140,7 @@ class TeachableIE(TeachableBaseIE):
     @staticmethod
     def _is_teachable(webpage):
         return 'teachableTracker.linker:autoLink' in webpage and re.search(
-            r'<link[^>]+href=["\']https?://process\.fs\.teachablecdn\.com',
+            r'<link[^>]+href=["\']https?://(?:process\.fs|assets)\.teachablecdn\.com',
             webpage)
 
     @staticmethod

From 794771a164009ff94046c98d8a7d45f7706547af Mon Sep 17 00:00:00 2001
From: JamKage <JamKage@users.noreply.github.com>
Date: Sun, 27 Dec 2020 17:36:21 +0000
Subject: [PATCH 007/860] [go] Added support for FXNetworks (#26826)

Co-authored-by: James Kirrage <james.kirrage@mortgagegym.com>

closes #13972
closes #22467
closes #23754
---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/fxnetworks.py | 77 ------------------------------
 youtube_dl/extractor/go.py         | 21 +++++++-
 3 files changed, 19 insertions(+), 80 deletions(-)
 delete mode 100644 youtube_dl/extractor/fxnetworks.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index da472d58e..cf50b897b 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -399,7 +399,6 @@ from .fujitv import FujiTVFODPlus7IE
 from .funimation import FunimationIE
 from .funk import FunkIE
 from .fusion import FusionIE
-from .fxnetworks import FXNetworksIE
 from .gaia import GaiaIE
 from .gameinformer import GameInformerIE
 from .gamespot import GameSpotIE
diff --git a/youtube_dl/extractor/fxnetworks.py b/youtube_dl/extractor/fxnetworks.py
deleted file mode 100644
index 00e67426b..000000000
--- a/youtube_dl/extractor/fxnetworks.py
+++ /dev/null
@@ -1,77 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-from .adobepass import AdobePassIE
-from ..utils import (
-    extract_attributes,
-    int_or_none,
-    parse_age_limit,
-    smuggle_url,
-    update_url_query,
-)
-
-
-class FXNetworksIE(AdobePassIE):
-    _VALID_URL = r'https?://(?:www\.)?(?:fxnetworks|simpsonsworld)\.com/video/(?P<id>\d+)'
-    _TESTS = [{
-        'url': 'http://www.fxnetworks.com/video/1032565827847',
-        'md5': '8d99b97b4aa7a202f55b6ed47ea7e703',
-        'info_dict': {
-            'id': 'dRzwHC_MMqIv',
-            'ext': 'mp4',
-            'title': 'First Look: Better Things - Season 2',
-            'description': 'Because real life is like a fart. Watch this FIRST LOOK to see what inspired the new season of Better Things.',
-            'age_limit': 14,
-            'uploader': 'NEWA-FNG-FX',
-            'upload_date': '20170825',
-            'timestamp': 1503686274,
-            'episode_number': 0,
-            'season_number': 2,
-            'series': 'Better Things',
-        },
-        'add_ie': ['ThePlatform'],
-    }, {
-        'url': 'http://www.simpsonsworld.com/video/716094019682',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        if 'The content you are trying to access is not available in your region.' in webpage:
-            self.raise_geo_restricted()
-        video_data = extract_attributes(self._search_regex(
-            r'(<a.+?rel="https?://link\.theplatform\.com/s/.+?</a>)', webpage, 'video data'))
-        player_type = self._search_regex(r'playerType\s*=\s*[\'"]([^\'"]+)', webpage, 'player type', default=None)
-        release_url = video_data['rel']
-        title = video_data['data-title']
-        rating = video_data.get('data-rating')
-        query = {
-            'mbr': 'true',
-        }
-        if player_type == 'movies':
-            query.update({
-                'manifest': 'm3u',
-            })
-        else:
-            query.update({
-                'switch': 'http',
-            })
-        if video_data.get('data-req-auth') == '1':
-            resource = self._get_mvpd_resource(
-                video_data['data-channel'], title,
-                video_data.get('data-guid'), rating)
-            query['auth'] = self._extract_mvpd_auth(url, video_id, 'fx', resource)
-
-        return {
-            '_type': 'url_transparent',
-            'id': video_id,
-            'title': title,
-            'url': smuggle_url(update_url_query(release_url, query), {'force_smil_url': True}),
-            'series': video_data.get('data-show-title'),
-            'episode_number': int_or_none(video_data.get('data-episode')),
-            'season_number': int_or_none(video_data.get('data-season')),
-            'thumbnail': video_data.get('data-large-thumb'),
-            'age_limit': parse_age_limit(rating),
-            'ie_key': 'ThePlatform',
-        }
diff --git a/youtube_dl/extractor/go.py b/youtube_dl/extractor/go.py
index 03cfba91f..0d731e90a 100644
--- a/youtube_dl/extractor/go.py
+++ b/youtube_dl/extractor/go.py
@@ -38,13 +38,17 @@ class GoIE(AdobePassIE):
         'disneynow': {
             'brand': '011',
             'resource_id': 'Disney',
-        }
+        },
+        'fxnow.fxnetworks': {
+            'brand': '025',
+            'requestor_id': 'dtci',
+        },
     }
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
                             (?:(?P<sub_domain>%s)\.)?go|
-                            (?P<sub_domain_2>abc|freeform|disneynow)
+                            (?P<sub_domain_2>abc|freeform|disneynow|fxnow\.fxnetworks)
                         )\.com/
                         (?:
                             (?:[^/]+/)*(?P<id>[Vv][Dd][Kk][Aa]\w+)|
@@ -99,6 +103,19 @@ class GoIE(AdobePassIE):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'https://fxnow.fxnetworks.com/shows/better-things/video/vdka12782841',
+        'info_dict': {
+            'id': 'VDKA12782841',
+            'ext': 'mp4',
+            'title': 'First Look: Better Things - Season 2',
+            'description': 'md5:fa73584a95761c605d9d54904e35b407',
+        },
+        'params': {
+            'geo_bypass_ip_block': '3.244.239.0/24',
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://abc.go.com/shows/the-catch/episode-guide/season-01/10-the-wedding',
         'only_matching': True,

From af93ecfd88d539cccea97f6cfc33b8cbe362a8ed Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 27 Dec 2020 22:26:20 +0100
Subject: [PATCH 008/860] [toggle] add support for live.mewatch.sg (closes
 #27555)

---
 youtube_dl/extractor/toggle.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/toggle.py b/youtube_dl/extractor/toggle.py
index 3b9b54759..270c84daa 100644
--- a/youtube_dl/extractor/toggle.py
+++ b/youtube_dl/extractor/toggle.py
@@ -200,7 +200,7 @@ class ToggleIE(InfoExtractor):
 
 class MeWatchIE(InfoExtractor):
     IE_NAME = 'mewatch'
-    _VALID_URL = r'https?://(?:www\.)?mewatch\.sg/watch/[^/?#&]+-(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:(?:www|live)\.)?mewatch\.sg/watch/[^/?#&]+-(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'https://www.mewatch.sg/watch/Recipe-Of-Life-E1-179371',
         'info_dict': {
@@ -220,6 +220,9 @@ class MeWatchIE(InfoExtractor):
     }, {
         'url': 'https://www.mewatch.sg/watch/Little-Red-Dot-Detectives-S2-%E6%90%9C%E5%AF%86%E3%80%82%E6%89%93%E5%8D%A1%E3%80%82%E5%B0%8F%E7%BA%A2%E7%82%B9-S2-E1-176232',
         'only_matching': True,
+    }, {
+        'url': 'https://live.mewatch.sg/watch/Recipe-Of-Life-E41-189759',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From c0071885987b3737d2c586133007c61ab513a477 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 27 Dec 2020 23:47:28 +0100
Subject: [PATCH 009/860] [zype] Add support for uplynk videos

---
 youtube_dl/extractor/zype.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/zype.py b/youtube_dl/extractor/zype.py
index 2e2e97a0c..5288f40d8 100644
--- a/youtube_dl/extractor/zype.py
+++ b/youtube_dl/extractor/zype.py
@@ -85,7 +85,13 @@ class ZypeIE(InfoExtractor):
         else:
             m3u8_url = self._search_regex(
                 r'(["\'])(?P<url>(?:(?!\1).)+\.m3u8(?:(?!\1).)*)\1',
-                body, 'm3u8 url', group='url')
+                body, 'm3u8 url', group='url', default=None)
+            if not m3u8_url:
+                source = self._parse_json(self._search_regex(
+                    r'(?s)sources\s*:\s*\[\s*({.+?})\s*\]', body,
+                    'source'), video_id, js_to_json)
+                if source.get('integration') == 'verizon-media':
+                    m3u8_url = 'https://content.uplynk.com/%s.m3u8' % source['id']
             formats = self._extract_m3u8_formats(
                 m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls')
             text_tracks = self._search_regex(

From f27224d57b6768569e1aedfaff326605bdb4f049 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 28 Dec 2020 10:50:29 +0100
Subject: [PATCH 010/860] [piksel] import format extraction

---
 youtube_dl/extractor/nhk.py    |   2 +-
 youtube_dl/extractor/piksel.py | 109 ++++++++++++++++++++++++---------
 2 files changed, 80 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/nhk.py b/youtube_dl/extractor/nhk.py
index c5b406573..8a9331a79 100644
--- a/youtube_dl/extractor/nhk.py
+++ b/youtube_dl/extractor/nhk.py
@@ -90,7 +90,7 @@ class NhkVodIE(NhkBaseIE):
     _TESTS = [{
         # video clip
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999011/',
-        'md5': '256a1be14f48d960a7e61e2532d95ec3',
+        'md5': '7a90abcfe610ec22a6bfe15bd46b30ca',
         'info_dict': {
             'id': 'a95j5iza',
             'ext': 'mp4',
diff --git a/youtube_dl/extractor/piksel.py b/youtube_dl/extractor/piksel.py
index 88b6859b0..ecf56ff8f 100644
--- a/youtube_dl/extractor/piksel.py
+++ b/youtube_dl/extractor/piksel.py
@@ -6,16 +6,33 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
-    ExtractorError,
     dict_get,
+    ExtractorError,
     int_or_none,
-    unescapeHTML,
     parse_iso8601,
+    try_get,
+    unescapeHTML,
 )
 
 
 class PikselIE(InfoExtractor):
-    _VALID_URL = r'https?://player\.piksel\.com/v/(?:refid/[^/]+/prefid/)?(?P<id>[a-z0-9_]+)'
+    _VALID_URL = r'''(?x)https?://
+        (?:
+            (?:
+                player\.
+                    (?:
+                        olympusattelecom|
+                        vibebyvista
+                    )|
+                (?:api|player)\.multicastmedia|
+                (?:api-ovp|player)\.piksel
+            )\.com|
+            (?:
+                mz-edge\.stream\.co|
+                movie-s\.nhk\.or
+            )\.jp|
+            vidego\.baltimorecity\.gov
+        )/v/(?:refid/(?P<refid>[^/]+)/prefid/)?(?P<id>[\w-]+)'''
     _TESTS = [
         {
             'url': 'http://player.piksel.com/v/ums2867l',
@@ -56,46 +73,41 @@ class PikselIE(InfoExtractor):
         if mobj:
             return mobj.group('url')
 
+    def _call_api(self, app_token, resource, display_id, query, fatal=True):
+        response = (self._download_json(
+            'http://player.piksel.com/ws/ws_%s/api/%s/mode/json/apiv/5' % (resource, app_token),
+            display_id, query=query, fatal=fatal) or {}).get('response')
+        failure = try_get(response, lambda x: x['failure']['reason'])
+        if failure:
+            if fatal:
+                raise ExtractorError(failure, expected=True)
+            self.report_warning(failure)
+        return response
+
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        ref_id, display_id = re.match(self._VALID_URL, url).groups()
         webpage = self._download_webpage(url, display_id)
-        video_id = self._search_regex(
-            r'data-de-program-uuid=[\'"]([a-z0-9]+)',
-            webpage, 'program uuid', default=display_id)
         app_token = self._search_regex([
             r'clientAPI\s*:\s*"([^"]+)"',
             r'data-de-api-key\s*=\s*"([^"]+)"'
         ], webpage, 'app token')
-        response = self._download_json(
-            'http://player.piksel.com/ws/ws_program/api/%s/mode/json/apiv/5' % app_token,
-            video_id, query={
-                'v': video_id
-            })['response']
-        failure = response.get('failure')
-        if failure:
-            raise ExtractorError(response['failure']['reason'], expected=True)
-        video_data = response['WsProgramResponse']['program']['asset']
+        query = {'refid': ref_id, 'prefid': display_id} if ref_id else {'v': display_id}
+        program = self._call_api(
+            app_token, 'program', display_id, query)['WsProgramResponse']['program']
+        video_id = program['uuid']
+        video_data = program['asset']
         title = video_data['title']
+        asset_type = dict_get(video_data, ['assetType', 'asset_type'])
 
         formats = []
 
-        m3u8_url = dict_get(video_data, [
-            'm3u8iPadURL',
-            'ipadM3u8Url',
-            'm3u8AndroidURL',
-            'm3u8iPhoneURL',
-            'iphoneM3u8Url'])
-        if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', 'm3u8_native',
-                m3u8_id='hls', fatal=False))
-
-        asset_type = dict_get(video_data, ['assetType', 'asset_type'])
-        for asset_file in video_data.get('assetFiles', []):
+        def process_asset_file(asset_file):
+            if not asset_file:
+                return
             # TODO: extract rtmp formats
             http_url = asset_file.get('http_url')
             if not http_url:
-                continue
+                return
             tbr = None
             vbr = int_or_none(asset_file.get('videoBitrate'), 1024)
             abr = int_or_none(asset_file.get('audioBitrate'), 1024)
@@ -118,6 +130,43 @@ class PikselIE(InfoExtractor):
                 'filesize': int_or_none(asset_file.get('filesize')),
                 'tbr': tbr,
             })
+
+        def process_asset_files(asset_files):
+            for asset_file in (asset_files or []):
+                process_asset_file(asset_file)
+
+        process_asset_files(video_data.get('assetFiles'))
+        process_asset_file(video_data.get('referenceFile'))
+        if not formats:
+            asset_id = video_data.get('assetid') or program.get('assetid')
+            if asset_id:
+                process_asset_files(try_get(self._call_api(
+                    app_token, 'asset_file', display_id, {
+                        'assetid': asset_id,
+                    }, False), lambda x: x['WsAssetFileResponse']['AssetFiles']))
+
+        m3u8_url = dict_get(video_data, [
+            'm3u8iPadURL',
+            'ipadM3u8Url',
+            'm3u8AndroidURL',
+            'm3u8iPhoneURL',
+            'iphoneM3u8Url'])
+        if m3u8_url:
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native',
+                m3u8_id='hls', fatal=False))
+
+        smil_url = dict_get(video_data, ['httpSmil', 'hdSmil', 'rtmpSmil'])
+        if smil_url:
+            transform_source = None
+            if ref_id == 'nhkworld':
+                # TODO: figure out if this is something to be fixed in urljoin,
+                # _parse_smil_formats or keep it here
+                transform_source = lambda x: x.replace('src="/', 'src="').replace('/media"', '/media/"')
+            formats.extend(self._extract_smil_formats(
+                re.sub(r'/od/[^/]+/', '/od/http/', smil_url), video_id,
+                transform_source=transform_source, fatal=False))
+
         self._sort_formats(formats)
 
         subtitles = {}

From 782ea947b487d4ef2b7b11fa40a00a518b529fed Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 28 Dec 2020 11:12:57 +0100
Subject: [PATCH 011/860] [brightcove] remove sonyliv specific code

---
 youtube_dl/extractor/brightcove.py | 8 --------
 1 file changed, 8 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 300d75458..65b44c099 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -534,14 +534,6 @@ class BrightcoveNewIE(AdobePassIE):
                         'format_id': build_format_id('rtmp'),
                     })
                 formats.append(f)
-        if not formats:
-            # for sonyliv.com DRM protected videos
-            s3_source_url = json_data.get('custom_fields', {}).get('s3sourceurl')
-            if s3_source_url:
-                formats.append({
-                    'url': s3_source_url,
-                    'format_id': 'source',
-                })
 
         errors = json_data.get('errors')
         if not formats and errors:

From 64e419bd7386b2a16a3c2e7ac5da30427afe856d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 28 Dec 2020 18:19:30 +0100
Subject: [PATCH 012/860] [aparat] Fix extraction

closes #22285
closes #22611
closes #23348
closes #24354
closes #24591
closes #24904
closes #25418
closes #26070
closes #26350
closes #26738
closes #27563
---
 youtube_dl/extractor/aparat.py | 20 +++++++-------------
 1 file changed, 7 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/aparat.py b/youtube_dl/extractor/aparat.py
index 883dcee7a..a9527e785 100644
--- a/youtube_dl/extractor/aparat.py
+++ b/youtube_dl/extractor/aparat.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    get_element_by_id,
     int_or_none,
     merge_dicts,
     mimetype2ext,
@@ -39,23 +40,15 @@ class AparatIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id, fatal=False)
 
         if not webpage:
-            # Note: There is an easier-to-parse configuration at
-            # http://www.aparat.com/video/video/config/videohash/%video_id
-            # but the URL in there does not work
             webpage = self._download_webpage(
                 'http://www.aparat.com/video/video/embed/vt/frame/showvideo/yes/videohash/' + video_id,
                 video_id)
 
-        options = self._parse_json(
-            self._search_regex(
-                r'options\s*=\s*JSON\.parse\(\s*(["\'])(?P<value>(?:(?!\1).)+)\1\s*\)',
-                webpage, 'options', group='value'),
-            video_id)
-
-        player = options['plugins']['sabaPlayerPlugin']
+        options = self._parse_json(self._search_regex(
+            r'options\s*=\s*({.+?})\s*;', webpage, 'options'), video_id)
 
         formats = []
-        for sources in player['multiSRC']:
+        for sources in (options.get('multiSRC') or []):
             for item in sources:
                 if not isinstance(item, dict):
                     continue
@@ -85,11 +78,12 @@ class AparatIE(InfoExtractor):
         info = self._search_json_ld(webpage, video_id, default={})
 
         if not info.get('title'):
-            info['title'] = player['title']
+            info['title'] = get_element_by_id('videoTitle', webpage) or \
+                self._html_search_meta(['og:title', 'twitter:title', 'DC.Title', 'title'], webpage, fatal=True)
 
         return merge_dicts(info, {
             'id': video_id,
             'thumbnail': url_or_none(options.get('poster')),
-            'duration': int_or_none(player.get('duration')),
+            'duration': int_or_none(options.get('duration')),
             'formats': formats,
         })

From f1bc56c99bac05dccb01c1b68ef778eb08fbcb71 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Dec 2020 02:11:48 +0700
Subject: [PATCH 013/860] [youtube:tab] Restore retry on browse requests
 (closes #27313, closes #27564)

---
 youtube_dl/extractor/youtube.py | 23 +++++++++++++++++++----
 1 file changed, 19 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5ef58d730..77f128285 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -16,6 +16,7 @@ from ..jsinterp import JSInterpreter
 from ..swfinterp import SWFInterpreter
 from ..compat import (
     compat_chr,
+    compat_HTTPError,
     compat_parse_qs,
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
@@ -3009,10 +3010,24 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
         for page_num in itertools.count(1):
             if not continuation:
                 break
-            browse = self._download_json(
-                'https://www.youtube.com/browse_ajax', None,
-                'Downloading page %d' % page_num,
-                headers=headers, query=continuation, fatal=False)
+            count = 0
+            retries = 3
+            while count <= retries:
+                try:
+                    # Downloading page may result in intermittent 5xx HTTP error
+                    # that is usually worked around with a retry
+                    browse = self._download_json(
+                        'https://www.youtube.com/browse_ajax', None,
+                        'Downloading page %d%s'
+                        % (page_num, ' (retry #%d)' % count if count else ''),
+                        headers=headers, query=continuation)
+                    break
+                except ExtractorError as e:
+                    if isinstance(e.cause, compat_HTTPError) and e.cause.code in (500, 503):
+                        count += 1
+                        if count <= retries:
+                            continue
+                    raise
             if not browse:
                 break
             response = try_get(browse, lambda x: x[1]['response'], dict)

From 71febd1c52d6de89ff571d4c212846aaaafb33ac Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Dec 2020 02:19:43 +0700
Subject: [PATCH 014/860] [youtube:tab] Improve URL matching (closes #27559)

---
 youtube_dl/extractor/youtube.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 77f128285..0044ed909 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1103,6 +1103,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
+            'only_matching': True,
+        },
     ]
 
     def __init__(self, *args, **kwargs):
@@ -2730,6 +2734,11 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
         'only_matching': True,
     }]
 
+    @classmethod
+    def suitable(cls, url):
+        return False if YoutubeIE.suitable(url) else super(
+            YoutubeTabIE, cls).suitable(url)
+
     def _extract_channel_id(self, webpage):
         channel_id = self._html_search_meta(
             'channelId', webpage, 'channel id', default=None)

From 1a95953867412bc7a785f21f6bff5145b2b13fd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Dec 2020 02:29:34 +0700
Subject: [PATCH 015/860] [youtube] Improve yt initial data extraction (closes
 #27524)

---
 youtube_dl/extractor/youtube.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0044ed909..87bdc1677 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -280,6 +280,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*=\s*({.+?})\s*;'
     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
+    _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
 
     def _call_api(self, ep, query, video_id):
         data = self._DEFAULT_API_DATA.copy()
@@ -297,7 +298,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     def _extract_yt_initial_data(self, video_id, webpage):
         return self._parse_json(
             self._search_regex(
-                (r'%s\s*\n' % self._YT_INITIAL_DATA_RE,
+                (r'%s\s*%s' % (self._YT_INITIAL_DATA_RE, self._YT_INITIAL_BOUNDARY_RE),
                  self._YT_INITIAL_DATA_RE), webpage, 'yt initial data'),
             video_id)
 
@@ -1103,6 +1104,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # another example of '};' in ytInitialData
+            'url': 'https://www.youtube.com/watch?v=gVfgbahppCY',
+            'only_matching': True,
+        },
         {
             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
             'only_matching': True,
@@ -1706,7 +1712,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if not video_info and not player_response:
             player_response = extract_player_response(
                 self._search_regex(
-                    (r'%s\s*(?:var\s+meta|</script|\n)' % self._YT_INITIAL_PLAYER_RESPONSE_RE,
+                    (r'%s\s*%s' % (self._YT_INITIAL_PLAYER_RESPONSE_RE, self._YT_INITIAL_BOUNDARY_RE),
                      self._YT_INITIAL_PLAYER_RESPONSE_RE), video_webpage,
                     'initial player response', default='{}'),
                 video_id)

From 38286ee729782e2d094c64190c7612663c40c2d7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Dec 2020 02:49:53 +0700
Subject: [PATCH 016/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 86b12b1c3..7e1bcb237 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,24 @@
+version <unreleased>
+
+Extractors
+* [youtube] Improve yt initial data extraction (#27524)
+* [youtube:tab] Improve URL matching #27559)
+* [youtube:tab] Restore retry on browse requests (#27313, #27564)
+* [aparat] Fix extraction (#22285, #22611, #23348, #24354, #24591, #24904,
+  #25418, #26070, #26350, #26738, #27563)
+- [brightcove] Remove sonyliv specific code
+* [piksel] Improve format extraction
++ [zype] Add support for uplynk videos
++ [toggle] Add support for live.mewatch.sg (#27555)
++ [go] Add support for fxnow.fxnetworks.com (#13972, #22467, #23754, #26826)
+* [teachable] Improve embed detection (#26923)
+* [mitele] Fix free video extraction (#24624, #25827, #26757)
+* [telecinco] Fix extraction
+* [youtube] Update invidious.snopyta.org (#22667)
+* [amcnetworks] Improve auth only video detection (#27548)
++ [generic] Add support for VHX Embeds (#27546)
+
+
 version 2020.12.26
 
 Extractors

From 479cc6d5a166dc2f250687616041c9f3b36c80b4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 29 Dec 2020 02:52:31 +0700
Subject: [PATCH 017/860] release 2020.12.29

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 docs/supportedsites.md                           | 1 -
 youtube_dl/version.py                            | 2 +-
 8 files changed, 14 insertions(+), 15 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 52295e426..0d5a8e666 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.29. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2020.12.26**
+- [ ] I've verified that I'm running youtube-dl version **2020.12.29**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2020.12.26
+ [debug] youtube-dl version 2020.12.29
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index 40a61bc80..88a6ea3fc 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.29. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2020.12.26**
+- [ ] I've verified that I'm running youtube-dl version **2020.12.29**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index e71c3ad61..1d75d1a79 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.29. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2020.12.26**
+- [ ] I've verified that I'm running youtube-dl version **2020.12.29**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index b10168073..83f1f43cd 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.29. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2020.12.26**
+- [ ] I've verified that I'm running youtube-dl version **2020.12.29**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2020.12.26
+ [debug] youtube-dl version 2020.12.29
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index cba0b3394..d3e03f78b 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.29. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2020.12.26**
+- [ ] I've verified that I'm running youtube-dl version **2020.12.29**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index 7e1bcb237..2dad14949 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2020.12.29
 
 Extractors
 * [youtube] Improve yt initial data extraction (#27524)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index c54507e90..1406ba8b8 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -317,7 +317,6 @@
  - **Funk**
  - **Fusion**
  - **Fux**
- - **FXNetworks**
  - **Gaia**
  - **GameInformer**
  - **GameSpot**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d1dcebd88..c7fb697c4 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2020.12.26'
+__version__ = '2020.12.29'

From bcfe485e0172ff32c450bb7835cfae7fca7594ae Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 29 Dec 2020 14:05:23 +0100
Subject: [PATCH 018/860] [brightcove] raise ExtractorError for DRM protected
 videos(closes #23467)(closes #27568)

---
 youtube_dl/extractor/brightcove.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/brightcove.py b/youtube_dl/extractor/brightcove.py
index 65b44c099..6022076ac 100644
--- a/youtube_dl/extractor/brightcove.py
+++ b/youtube_dl/extractor/brightcove.py
@@ -471,13 +471,18 @@ class BrightcoveNewIE(AdobePassIE):
     def _parse_brightcove_metadata(self, json_data, video_id, headers={}):
         title = json_data['name'].strip()
 
+        num_drm_sources = 0
         formats = []
-        for source in json_data.get('sources', []):
+        sources = json_data.get('sources') or []
+        for source in sources:
             container = source.get('container')
             ext = mimetype2ext(source.get('type'))
             src = source.get('src')
             # https://support.brightcove.com/playback-api-video-fields-reference#key_systems_object
-            if ext == 'ism' or container == 'WVM' or source.get('key_systems'):
+            if container == 'WVM' or source.get('key_systems'):
+                num_drm_sources += 1
+                continue
+            elif ext == 'ism':
                 continue
             elif ext == 'm3u8' or container == 'M2TS':
                 if not src:
@@ -535,11 +540,14 @@ class BrightcoveNewIE(AdobePassIE):
                     })
                 formats.append(f)
 
-        errors = json_data.get('errors')
-        if not formats and errors:
-            error = errors[0]
-            raise ExtractorError(
-                error.get('message') or error.get('error_subcode') or error['error_code'], expected=True)
+        if not formats:
+            errors = json_data.get('errors')
+            if errors:
+                error = errors[0]
+                raise ExtractorError(
+                    error.get('message') or error.get('error_subcode') or error['error_code'], expected=True)
+            if sources and num_drm_sources == len(sources):
+                raise ExtractorError('This video is DRM protected.', expected=True)
 
         self._sort_formats(formats)
 

From 7acd042bbb555962f42fa4f0f236772194d2da64 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 29 Dec 2020 14:06:58 +0100
Subject: [PATCH 019/860] [tenplay] fix format extraction(closes #26653)

---
 youtube_dl/extractor/tenplay.py | 34 ++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/tenplay.py b/youtube_dl/extractor/tenplay.py
index af325fea8..cd30d57f4 100644
--- a/youtube_dl/extractor/tenplay.py
+++ b/youtube_dl/extractor/tenplay.py
@@ -3,9 +3,10 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    HEADRequest,
     parse_age_limit,
     parse_iso8601,
-    smuggle_url,
+    # smuggle_url,
 )
 
 
@@ -24,14 +25,16 @@ class TenPlayIE(InfoExtractor):
             'uploader_id': '2199827728001',
         },
         'params': {
-            'format': 'bestvideo',
+            # 'format': 'bestvideo',
             'skip_download': True,
         }
     }, {
         'url': 'https://10play.com.au/how-to-stay-married/web-extras/season-1/terrys-talks-ep-1-embracing-change/tpv190915ylupc',
         'only_matching': True,
     }]
-    BRIGHTCOVE_URL_TEMPLATE = 'https://players.brightcove.net/2199827728001/cN6vRtRQt_default/index.html?videoId=%s'
+    # BRIGHTCOVE_URL_TEMPLATE = 'https://players.brightcove.net/2199827728001/cN6vRtRQt_default/index.html?videoId=%s'
+    _GEO_BYPASS = False
+    _FASTLY_URL_TEMPL = 'https://10-selector.global.ssl.fastly.net/s/kYEXFC/media/%s?mbr=true&manifest=m3u&format=redirect'
 
     def _real_extract(self, url):
         content_id = self._match_id(url)
@@ -40,19 +43,28 @@ class TenPlayIE(InfoExtractor):
         video = data.get('video') or {}
         metadata = data.get('metaData') or {}
         brightcove_id = video.get('videoId') or metadata['showContentVideoId']
-        brightcove_url = smuggle_url(
-            self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id,
-            {'geo_countries': ['AU']})
+        # brightcove_url = smuggle_url(
+        #     self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id,
+        #     {'geo_countries': ['AU']})
+        m3u8_url = self._request_webpage(HEADRequest(
+            self._FASTLY_URL_TEMPL % brightcove_id), brightcove_id).geturl()
+        if '10play-not-in-oz' in m3u8_url:
+            self.raise_geo_restricted(countries=['AU'])
+        formats = self._extract_m3u8_formats(m3u8_url, brightcove_id, 'mp4')
+        self._sort_formats(formats)
 
         return {
-            '_type': 'url_transparent',
-            'url': brightcove_url,
-            'id': content_id,
-            'title': video.get('title') or metadata.get('pageContentName') or metadata.get('showContentName'),
+            # '_type': 'url_transparent',
+            # 'url': brightcove_url,
+            'formats': formats,
+            'id': brightcove_id,
+            'title': video.get('title') or metadata.get('pageContentName') or metadata['showContentName'],
             'description': video.get('description'),
             'age_limit': parse_age_limit(video.get('showRatingClassification') or metadata.get('showProgramClassification')),
             'series': metadata.get('showName'),
             'season': metadata.get('showContentSeason'),
             'timestamp': parse_iso8601(metadata.get('contentPublishDate') or metadata.get('pageContentPublishDate')),
-            'ie_key': 'BrightcoveNew',
+            'thumbnail': video.get('poster'),
+            'uploader_id': '2199827728001',
+            # 'ie_key': 'BrightcoveNew',
         }

From c931c4b8ddb32371cddf48ea52d0c036a6a66240 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 29 Dec 2020 14:09:10 +0100
Subject: [PATCH 020/860] [sevenplay] detect API errors

---
 youtube_dl/extractor/sevenplus.py | 32 ++++++++++++++++++++-----------
 1 file changed, 21 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/sevenplus.py b/youtube_dl/extractor/sevenplus.py
index 84568ac69..240afc18f 100644
--- a/youtube_dl/extractor/sevenplus.py
+++ b/youtube_dl/extractor/sevenplus.py
@@ -4,8 +4,12 @@ from __future__ import unicode_literals
 import re
 
 from .brightcove import BrightcoveNewIE
-from ..compat import compat_str
+from ..compat import (
+    compat_HTTPError,
+    compat_str,
+)
 from ..utils import (
+    ExtractorError,
     try_get,
     update_url_query,
 )
@@ -41,16 +45,22 @@ class SevenPlusIE(BrightcoveNewIE):
     def _real_extract(self, url):
         path, episode_id = re.match(self._VALID_URL, url).groups()
 
-        media = self._download_json(
-            'https://videoservice.swm.digital/playback', episode_id, query={
-                'appId': '7plus',
-                'deviceType': 'web',
-                'platformType': 'web',
-                'accountId': 5303576322001,
-                'referenceId': 'ref:' + episode_id,
-                'deliveryId': 'csai',
-                'videoType': 'vod',
-            })['media']
+        try:
+            media = self._download_json(
+                'https://videoservice.swm.digital/playback', episode_id, query={
+                    'appId': '7plus',
+                    'deviceType': 'web',
+                    'platformType': 'web',
+                    'accountId': 5303576322001,
+                    'referenceId': 'ref:' + episode_id,
+                    'deliveryId': 'csai',
+                    'videoType': 'vod',
+                })['media']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                raise ExtractorError(self._parse_json(
+                    e.cause.read().decode(), episode_id)[0]['error_code'], expected=True)
+            raise
 
         for source in media.get('sources', {}):
             src = source.get('src')

From 53528e1d2385494c72349f609907f0164d6f8431 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 29 Dec 2020 14:10:46 +0100
Subject: [PATCH 021/860] [uktvplay] match new video URLs(closes #17909)

---
 youtube_dl/extractor/uktvplay.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/uktvplay.py b/youtube_dl/extractor/uktvplay.py
index 2137502a1..f28fd514d 100644
--- a/youtube_dl/extractor/uktvplay.py
+++ b/youtube_dl/extractor/uktvplay.py
@@ -5,10 +5,9 @@ from .common import InfoExtractor
 
 
 class UKTVPlayIE(InfoExtractor):
-    _VALID_URL = r'https?://uktvplay\.uktv\.co\.uk/.+?\?.*?\bvideo=(?P<id>\d+)'
-    _TEST = {
+    _VALID_URL = r'https?://uktvplay\.uktv\.co\.uk/(?:.+?\?.*?\bvideo=|([^/]+/)*watch-online/)(?P<id>\d+)'
+    _TESTS = [{
         'url': 'https://uktvplay.uktv.co.uk/shows/world-at-war/c/200/watch-online/?video=2117008346001',
-        'md5': '',
         'info_dict': {
             'id': '2117008346001',
             'ext': 'mp4',
@@ -23,7 +22,11 @@ class UKTVPlayIE(InfoExtractor):
             'skip_download': True,
         },
         'expected_warnings': ['Failed to download MPD manifest']
-    }
+    }, {
+        'url': 'https://uktvplay.uktv.co.uk/shows/africa/watch-online/5983349675001',
+        'only_matching': True,
+    }]
+    # BRIGHTCOVE_URL_TEMPLATE = 'https://players.brightcove.net/1242911124001/OrCyvJ2gyL_default/index.html?videoId=%s'
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1242911124001/H1xnMOqP_default/index.html?videoId=%s'
 
     def _real_extract(self, url):

From 9ee984fc760c9a3f4818055ff28ea886aadc58cb Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 29 Dec 2020 16:13:36 +0100
Subject: [PATCH 022/860] [aenetworks] add support for biography.com (closes
 #3863)

---
 youtube_dl/extractor/aenetworks.py | 97 ++++++++++++++++++++++--------
 youtube_dl/extractor/extractors.py |  2 +
 2 files changed, 73 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 3d0cf1208..237012978 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -6,6 +6,7 @@ import re
 from .theplatform import ThePlatformIE
 from ..utils import (
     ExtractorError,
+    GeoRestrictedError,
     int_or_none,
     update_url_query,
     urlencode_postdata,
@@ -28,6 +29,7 @@ class AENetworksBaseIE(ThePlatformIE):
         'lifetimemovieclub.com': ('LIFETIMEMOVIECLUB', 'lmc'),
         'fyi.tv': ('FYI', 'fyi'),
         'historyvault.com': (None, 'historyvault'),
+        'biography.com': (None, 'biography'),
     }
 
     def _extract_aen_smil(self, smil_url, video_id, auth=None):
@@ -54,6 +56,8 @@ class AENetworksBaseIE(ThePlatformIE):
                 tp_formats, tp_subtitles = self._extract_theplatform_smil(
                     m_url, video_id, 'Downloading %s SMIL data' % (q.get('switch') or q['assetTypes']))
             except ExtractorError as e:
+                if isinstance(e, GeoRestrictedError):
+                    raise
                 last_e = e
                 continue
             formats.extend(tp_formats)
@@ -67,6 +71,34 @@ class AENetworksBaseIE(ThePlatformIE):
             'subtitles': subtitles,
         }
 
+    def _extract_aetn_info(self, domain, filter_key, filter_value, url):
+        requestor_id, brand = self._DOMAIN_MAP[domain]
+        result = self._download_json(
+            'https://feeds.video.aetnd.com/api/v2/%s/videos' % brand,
+            filter_value, query={'filter[%s]' % filter_key: filter_value})['results'][0]
+        title = result['title']
+        video_id = result['id']
+        media_url = result['publicUrl']
+        theplatform_metadata = self._download_theplatform_metadata(self._search_regex(
+            r'https?://link\.theplatform\.com/s/([^?]+)', media_url, 'theplatform_path'), video_id)
+        info = self._parse_theplatform_metadata(theplatform_metadata)
+        auth = None
+        if theplatform_metadata.get('AETN$isBehindWall'):
+            resource = self._get_mvpd_resource(
+                requestor_id, theplatform_metadata['title'],
+                theplatform_metadata.get('AETN$PPL_pplProgramId') or theplatform_metadata.get('AETN$PPL_pplProgramId_OLD'),
+                theplatform_metadata['ratings'][0]['rating'])
+            auth = self._extract_mvpd_auth(
+                url, video_id, requestor_id, resource)
+        info.update(self._extract_aen_smil(media_url, video_id, auth))
+        info.update({
+            'title': title,
+            'series': result.get('seriesName'),
+            'season_number': int_or_none(result.get('tvSeasonNumber')),
+            'episode_number': int_or_none(result.get('tvSeasonEpisodeNumber')),
+        })
+        return info
+
 
 class AENetworksIE(AENetworksBaseIE):
     IE_NAME = 'aenetworks'
@@ -139,32 +171,7 @@ class AENetworksIE(AENetworksBaseIE):
 
     def _real_extract(self, url):
         domain, canonical = re.match(self._VALID_URL, url).groups()
-        requestor_id, brand = self._DOMAIN_MAP[domain]
-        result = self._download_json(
-            'https://feeds.video.aetnd.com/api/v2/%s/videos' % brand,
-            canonical, query={'filter[canonical]': '/' + canonical})['results'][0]
-        title = result['title']
-        video_id = result['id']
-        media_url = result['publicUrl']
-        theplatform_metadata = self._download_theplatform_metadata(self._search_regex(
-            r'https?://link\.theplatform\.com/s/([^?]+)', media_url, 'theplatform_path'), video_id)
-        info = self._parse_theplatform_metadata(theplatform_metadata)
-        auth = None
-        if theplatform_metadata.get('AETN$isBehindWall'):
-            resource = self._get_mvpd_resource(
-                requestor_id, theplatform_metadata['title'],
-                theplatform_metadata.get('AETN$PPL_pplProgramId') or theplatform_metadata.get('AETN$PPL_pplProgramId_OLD'),
-                theplatform_metadata['ratings'][0]['rating'])
-            auth = self._extract_mvpd_auth(
-                url, video_id, requestor_id, resource)
-        info.update(self._extract_aen_smil(media_url, video_id, auth))
-        info.update({
-            'title': title,
-            'series': result.get('seriesName'),
-            'season_number': int_or_none(result.get('tvSeasonNumber')),
-            'episode_number': int_or_none(result.get('tvSeasonEpisodeNumber')),
-        })
-        return info
+        return self._extract_aetn_info(domain, 'canonical', '/' + canonical, url)
 
 
 class AENetworksListBaseIE(AENetworksBaseIE):
@@ -294,3 +301,41 @@ class HistoryTopicIE(AENetworksBaseIE):
         return self.url_result(
             'http://www.history.com/videos/' + display_id,
             AENetworksIE.ie_key())
+
+
+class HistoryPlayerIE(AENetworksBaseIE):
+    IE_NAME = 'history:player'
+    _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:history|biography)\.com)/player/(?P<id>\d+)'
+
+    def _real_extract(self, url):
+        domain, video_id = re.match(self._VALID_URL, url).groups()
+        return self._extract_aetn_info(domain, 'id', video_id, url)
+
+
+class BiographyIE(AENetworksBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?biography\.com/video/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://www.biography.com/video/vincent-van-gogh-full-episode-2075049808',
+        'info_dict': {
+            'id': '30322987',
+            'ext': 'mp4',
+            'title': 'Vincent Van Gogh - Full Episode',
+            'description': 'A full biography about the most influential 20th century painter, Vincent Van Gogh.',
+            'timestamp': 1311970571,
+            'upload_date': '20110729',
+            'uploader': 'AENE-NEW',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+        'add_ie': ['ThePlatform'],
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        player_url = self._search_regex(
+            r'<phoenix-iframe[^>]+src="(%s)' % HistoryPlayerIE._VALID_URL,
+            webpage, 'player URL')
+        return self.url_result(player_url, HistoryPlayerIE.ie_key())
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index cf50b897b..20472f2f7 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -33,6 +33,8 @@ from .aenetworks import (
     AENetworksCollectionIE,
     AENetworksShowIE,
     HistoryTopicIE,
+    HistoryPlayerIE,
+    BiographyIE,
 )
 from .afreecatv import AfreecaTVIE
 from .airmozilla import AirMozillaIE

From 5966095e65b5365e8e4d211ffca6ab50514d3c84 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 29 Dec 2020 16:59:31 +0100
Subject: [PATCH 023/860] [aenetworks] fix HistoryPlayerIE tests

---
 youtube_dl/extractor/aenetworks.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 237012978..8e4963131 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -306,6 +306,7 @@ class HistoryTopicIE(AENetworksBaseIE):
 class HistoryPlayerIE(AENetworksBaseIE):
     IE_NAME = 'history:player'
     _VALID_URL = r'https?://(?:www\.)?(?P<domain>(?:history|biography)\.com)/player/(?P<id>\d+)'
+    _TESTS = []
 
     def _real_extract(self, url):
         domain, video_id = re.match(self._VALID_URL, url).groups()

From ebdcf70b0d52fd7b4627b2435b3081c7f4f4ce4b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 29 Dec 2020 17:15:13 +0100
Subject: [PATCH 024/860] [nbc] fix NBCSport VPlayer URL extraction(closes
 #16640)

---
 youtube_dl/extractor/nbc.py | 26 +++++++++++++++++++-------
 1 file changed, 19 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index ea5f5a315..9695a9616 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -158,7 +158,8 @@ class NBCIE(AdobePassIE):
 
 
 class NBCSportsVPlayerIE(InfoExtractor):
-    _VALID_URL = r'https?://vplayer\.nbcsports\.com/(?:[^/]+/)+(?P<id>[0-9a-zA-Z_]+)'
+    _VALID_URL_BASE = r'https?://(?:vplayer\.nbcsports\.com|(?:www\.)?nbcsports\.com/vplayer)/'
+    _VALID_URL = _VALID_URL_BASE + r'(?:[^/]+/)+(?P<id>[0-9a-zA-Z_]+)'
 
     _TESTS = [{
         'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_embed/select/9CsDKds0kvHI',
@@ -174,12 +175,15 @@ class NBCSportsVPlayerIE(InfoExtractor):
     }, {
         'url': 'https://vplayer.nbcsports.com/p/BxmELC/nbcsports_embed/select/media/_hqLjQ95yx8Z',
         'only_matching': True,
+    }, {
+        'url': 'https://www.nbcsports.com/vplayer/p/BxmELC/nbcsports/select/PHJSaFWbrTY9?form=html&autoPlay=true',
+        'only_matching': True,
     }]
 
     @staticmethod
     def _extract_url(webpage):
         iframe_m = re.search(
-            r'<iframe[^>]+src="(?P<url>https?://vplayer\.nbcsports\.com/[^"]+)"', webpage)
+            r'<(?:iframe[^>]+|div[^>]+data-(?:mpx-)?)src="(?P<url>%s[^"]+)"' % NBCSportsVPlayerIE._VALID_URL_BASE, webpage)
         if iframe_m:
             return iframe_m.group('url')
 
@@ -192,21 +196,29 @@ class NBCSportsVPlayerIE(InfoExtractor):
 
 
 class NBCSportsIE(InfoExtractor):
-    # Does not include https because its certificate is invalid
-    _VALID_URL = r'https?://(?:www\.)?nbcsports\.com//?(?:[^/]+/)+(?P<id>[0-9a-z-]+)'
+    _VALID_URL = r'https?://(?:www\.)?nbcsports\.com//?(?!vplayer/)(?:[^/]+/)+(?P<id>[0-9a-z-]+)'
 
-    _TEST = {
+    _TESTS = [{
+        # iframe src
         'url': 'http://www.nbcsports.com//college-basketball/ncaab/tom-izzo-michigan-st-has-so-much-respect-duke',
         'info_dict': {
             'id': 'PHJSaFWbrTY9',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Tom Izzo, Michigan St. has \'so much respect\' for Duke',
             'description': 'md5:ecb459c9d59e0766ac9c7d5d0eda8113',
             'uploader': 'NBCU-SPORTS',
             'upload_date': '20150330',
             'timestamp': 1427726529,
         }
-    }
+    }, {
+        # data-mpx-src
+        'url': 'https://www.nbcsports.com/philadelphia/philadelphia-phillies/bruce-bochy-hector-neris-hes-idiot',
+        'only_matching': True,
+    }, {
+        # data-src
+        'url': 'https://www.nbcsports.com/boston/video/report-card-pats-secondary-no-match-josh-allen',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)

From c706fbe9fe4a517ad43ca95393384e5f78870f82 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 29 Dec 2020 17:21:05 +0100
Subject: [PATCH 025/860] [nbc] Remove CSNNE extractor

---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/nbc.py        | 27 ---------------------------
 2 files changed, 28 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 20472f2f7..51e6a463a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -692,7 +692,6 @@ from .nba import (
     NBAChannelIE,
 )
 from .nbc import (
-    CSNNEIE,
     NBCIE,
     NBCNewsIE,
     NBCOlympicsIE,
diff --git a/youtube_dl/extractor/nbc.py b/youtube_dl/extractor/nbc.py
index 9695a9616..0d77648c2 100644
--- a/youtube_dl/extractor/nbc.py
+++ b/youtube_dl/extractor/nbc.py
@@ -286,33 +286,6 @@ class NBCSportsStreamIE(AdobePassIE):
         }
 
 
-class CSNNEIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?csnne\.com/video/(?P<id>[0-9a-z-]+)'
-
-    _TEST = {
-        'url': 'http://www.csnne.com/video/snc-evening-update-wright-named-red-sox-no-5-starter',
-        'info_dict': {
-            'id': 'yvBLLUgQ8WU0',
-            'ext': 'mp4',
-            'title': 'SNC evening update: Wright named Red Sox\' No. 5 starter.',
-            'description': 'md5:1753cfee40d9352b19b4c9b3e589b9e3',
-            'timestamp': 1459369979,
-            'upload_date': '20160330',
-            'uploader': 'NBCU-SPORTS',
-        }
-    }
-
-    def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        return {
-            '_type': 'url_transparent',
-            'ie_key': 'ThePlatform',
-            'url': self._html_search_meta('twitter:player:stream', webpage),
-            'display_id': display_id,
-        }
-
-
 class NBCNewsIE(ThePlatformIE):
     _VALID_URL = r'(?x)https?://(?:www\.)?(?:nbcnews|today|msnbc)\.com/([^/]+/)*(?:.*-)?(?P<id>[^/?]+)'
 

From 9c1e164e0cd77331ea4f0b474b32fd06f84bad71 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 29 Dec 2020 19:29:08 +0100
Subject: [PATCH 026/860] [YoutubeDL] Allow format filtering using audio
 language(#16209)

---
 README.md               | 1 +
 youtube_dl/YoutubeDL.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index ffa34493e..85fed6d3a 100644
--- a/README.md
+++ b/README.md
@@ -678,6 +678,7 @@ Also filtering work for comparisons `=` (equals), `^=` (starts with), `$=` (ends
  - `container`: Name of the container format
  - `protocol`: The protocol that will be used for the actual download, lower-case (`http`, `https`, `rtsp`, `rtmp`, `rtmpe`, `mms`, `f4m`, `ism`, `http_dash_segments`, `m3u8`, or `m3u8_native`)
  - `format_id`: A short description of the format
+ - `language`: Language code
 
 Any string comparison may be prefixed with negation `!` in order to produce an opposite comparison, e.g. `!*=` (does not contain).
 
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6f477bc32..aaac149e9 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1083,7 +1083,7 @@ class YoutubeDL(object):
                 '*=': lambda attr, value: value in attr,
             }
             str_operator_rex = re.compile(r'''(?x)
-                \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id)
+                \s*(?P<key>ext|acodec|vcodec|container|protocol|format_id|language)
                 \s*(?P<negation>!\s*)?(?P<op>%s)(?P<none_inclusive>\s*\?)?
                 \s*(?P<value>[a-zA-Z0-9._-]+)
                 \s*$

From 9dd674e1d20440564a3d25f33cd8785695e110f6 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 30 Dec 2020 09:22:30 +0100
Subject: [PATCH 027/860] [utils] accept only supported protocols in
 url_or_none

---
 test/test_utils.py  | 5 +++++
 youtube_dl/utils.py | 2 +-
 2 files changed, 6 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 925a21d34..d49d3239c 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -554,6 +554,11 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(url_or_none('http$://foo.de'), None)
         self.assertEqual(url_or_none('http://foo.de'), 'http://foo.de')
         self.assertEqual(url_or_none('//foo.de'), '//foo.de')
+        self.assertEqual(url_or_none('s3://foo.de'), None)
+        self.assertEqual(url_or_none('rtmpte://foo.de'), 'rtmpte://foo.de')
+        self.assertEqual(url_or_none('mms://foo.de'), 'mms://foo.de')
+        self.assertEqual(url_or_none('rtspu://foo.de'), 'rtspu://foo.de')
+        self.assertEqual(url_or_none('ftps://foo.de'), 'ftps://foo.de')
 
     def test_parse_age_limit(self):
         self.assertEqual(parse_age_limit(None), None)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 8cefafd79..d5fb6fd24 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3640,7 +3640,7 @@ def url_or_none(url):
     if not url or not isinstance(url, compat_str):
         return None
     url = url.strip()
-    return url if re.match(r'^(?:[a-zA-Z][\da-zA-Z.+-]*:)?//', url) else None
+    return url if re.match(r'^(?:(?:https?|rt(?:m(?:pt?[es]?|fp)|sp[su]?)|mms|ftps?):)?//', url) else None
 
 
 def parse_duration(s):

From f7e95fb2a0516f90edffe72d9911222d1ed1a2bc Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 30 Dec 2020 09:24:37 +0100
Subject: [PATCH 028/860] [yandexvideo] fix extraction(closes #25000)

---
 youtube_dl/extractor/yandexvideo.py | 116 ++++++++++++++++++----------
 1 file changed, 76 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/yandexvideo.py b/youtube_dl/extractor/yandexvideo.py
index 46529be05..36d01cc8e 100644
--- a/youtube_dl/extractor/yandexvideo.py
+++ b/youtube_dl/extractor/yandexvideo.py
@@ -13,26 +13,30 @@ class YandexVideoIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
-                            yandex\.ru(?:/portal/(?:video|efir))?/?\?.*?stream_id=|
+                            yandex\.ru(?:/(?:portal/(?:video|efir)|efir))?/?\?.*?stream_id=|
                             frontend\.vh\.yandex\.ru/player/
                         )
-                        (?P<id>[\da-f]+)
+                        (?P<id>(?:[\da-f]{32}|[\w-]{12}))
                     '''
     _TESTS = [{
-        'url': 'https://yandex.ru/portal/video?stream_id=4dbb262b4fe5cf15a215de4f34eee34d',
-        'md5': '33955d7ae052f15853dc41f35f17581c',
+        'url': 'https://yandex.ru/portal/video?stream_id=4dbb36ec4e0526d58f9f2dc8f0ecf374',
+        'md5': 'e02a05bfaf0d9615ef07ae3a10f4faf4',
         'info_dict': {
-            'id': '4dbb262b4fe5cf15a215de4f34eee34d',
+            'id': '4dbb36ec4e0526d58f9f2dc8f0ecf374',
             'ext': 'mp4',
-            'title': 'В Нью-Йорке баржи и теплоход оторвались от причала и расплылись по Гудзону',
-            'description': '',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'timestamp': 0,
-            'duration': 30,
+            'title': 'Русский Вудсток - главный рок-фест в истории СССР / вДудь',
+            'description': 'md5:7d6b8d4bc4a3b9a56499916c1ea5b5fa',
+            'thumbnail': r're:^https?://',
+            'timestamp': 1549972939,
+            'duration': 5575,
             'age_limit': 18,
+            'upload_date': '20190212',
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
         },
     }, {
-        'url': 'https://yandex.ru/portal/efir?stream_id=4dbb36ec4e0526d58f9f2dc8f0ecf374&from=morda',
+        'url': 'https://yandex.ru/portal/efir?stream_id=4dbb262b4fe5cf15a215de4f34eee34d&from=morda',
         'only_matching': True,
     }, {
         'url': 'https://yandex.ru/?stream_id=4dbb262b4fe5cf15a215de4f34eee34d',
@@ -52,53 +56,85 @@ class YandexVideoIE(InfoExtractor):
         # DASH with DRM
         'url': 'https://yandex.ru/portal/video?from=morda&stream_id=485a92d94518d73a9d0ff778e13505f8',
         'only_matching': True,
+    }, {
+        'url': 'https://yandex.ru/efir?stream_active=watching&stream_id=v7a2dZ-v5mSI&from_block=efir_newtab',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         content = self._download_json(
-            'https://frontend.vh.yandex.ru/v22/player/%s.json' % video_id,
-            video_id, query={
-                'stream_options': 'hires',
-                'disable_trackings': 1,
-            })['content']
+            # 'https://frontend.vh.yandex.ru/v23/player/%s.json' % video_id,
+            # video_id, query={
+            #     'stream_options': 'hires',
+            #     'disable_trackings': 1,
+            # })['content']
+            'https://frontend.vh.yandex.ru/graphql', video_id, data=b'''{
+  player(content_id: "%s") {
+    computed_title
+    content_url
+    description
+    dislikes
+    duration
+    likes
+    program_title
+    release_date
+    release_date_ut
+    release_year
+    restriction_age
+    season
+    start_time
+    streams
+    thumbnail
+    title
+    views_count
+  }
+}''' % video_id.encode())['player']['content']['content']
 
-        content_url = url_or_none(content.get('content_url')) or url_or_none(
-            content['streams'][0]['url'])
-        title = content.get('title') or content.get('computed_title')
+        title = content.get('title') or content['computed_title']
 
-        ext = determine_ext(content_url)
-
-        if ext == 'm3u8':
-            formats = self._extract_m3u8_formats(
-                content_url, video_id, 'mp4', entry_protocol='m3u8_native',
-                m3u8_id='hls')
-        elif ext == 'mpd':
-            formats = self._extract_mpd_formats(
-                content_url, video_id, mpd_id='dash')
-        else:
-            formats = [{'url': content_url}]
+        formats = []
+        streams = content.get('streams') or []
+        streams.append({'url': content.get('content_url')})
+        for stream in streams:
+            content_url = url_or_none(stream.get('url'))
+            if not content_url:
+                continue
+            ext = determine_ext(content_url)
+            if ext == 'ismc':
+                continue
+            elif ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    content_url, video_id, 'mp4',
+                    'm3u8_native', m3u8_id='hls', fatal=False))
+            elif ext == 'mpd':
+                formats.extend(self._extract_mpd_formats(
+                    content_url, video_id, mpd_id='dash', fatal=False))
+            else:
+                formats.append({'url': content_url})
 
         self._sort_formats(formats)
 
-        description = content.get('description')
-        thumbnail = content.get('thumbnail')
         timestamp = (int_or_none(content.get('release_date'))
                      or int_or_none(content.get('release_date_ut'))
                      or int_or_none(content.get('start_time')))
-        duration = int_or_none(content.get('duration'))
-        series = content.get('program_title')
-        age_limit = int_or_none(content.get('restriction_age'))
+        season = content.get('season') or {}
 
         return {
             'id': video_id,
             'title': title,
-            'description': description,
-            'thumbnail': thumbnail,
+            'description': content.get('description'),
+            'thumbnail': content.get('thumbnail'),
             'timestamp': timestamp,
-            'duration': duration,
-            'series': series,
-            'age_limit': age_limit,
+            'duration': int_or_none(content.get('duration')),
+            'series': content.get('program_title'),
+            'age_limit': int_or_none(content.get('restriction_age')),
+            'view_count': int_or_none(content.get('views_count')),
+            'like_count': int_or_none(content.get('likes')),
+            'dislike_count': int_or_none(content.get('dislikes')),
+            'season_number': int_or_none(season.get('season_number')),
+            'season_id': season.get('id'),
+            'release_year': int_or_none(content.get('release_year')),
             'formats': formats,
         }

From bdd044e67b5d10736aa712e9be64beff0d47f490 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 30 Dec 2020 13:30:11 +0100
Subject: [PATCH 029/860] [yandexvideo] use old api call as fallback

---
 youtube_dl/extractor/yandexvideo.py | 18 +++++++++++-------
 1 file changed, 11 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/yandexvideo.py b/youtube_dl/extractor/yandexvideo.py
index 36d01cc8e..ab8c84c93 100644
--- a/youtube_dl/extractor/yandexvideo.py
+++ b/youtube_dl/extractor/yandexvideo.py
@@ -5,6 +5,7 @@ from .common import InfoExtractor
 from ..utils import (
     determine_ext,
     int_or_none,
+    try_get,
     url_or_none,
 )
 
@@ -64,12 +65,7 @@ class YandexVideoIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        content = self._download_json(
-            # 'https://frontend.vh.yandex.ru/v23/player/%s.json' % video_id,
-            # video_id, query={
-            #     'stream_options': 'hires',
-            #     'disable_trackings': 1,
-            # })['content']
+        player = try_get((self._download_json(
             'https://frontend.vh.yandex.ru/graphql', video_id, data=b'''{
   player(content_id: "%s") {
     computed_title
@@ -90,7 +86,15 @@ class YandexVideoIE(InfoExtractor):
     title
     views_count
   }
-}''' % video_id.encode())['player']['content']['content']
+}''' % video_id.encode(), fatal=False)), lambda x: x['player']['content'])
+        if not player or player.get('error'):
+            player = self._download_json(
+                'https://frontend.vh.yandex.ru/v23/player/%s.json' % video_id,
+                video_id, query={
+                    'stream_options': 'hires',
+                    'disable_trackings': 1,
+                })
+        content = player['content']
 
         title = content.get('title') or content['computed_title']
 

From bd18824c2a99d6d01b00edfa186b9fd227af255c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 30 Dec 2020 13:43:56 +0100
Subject: [PATCH 030/860] [yandexdisk] fix extraction(closes #17861)(closes
 #27131)

---
 youtube_dl/extractor/yandexdisk.py | 138 +++++++++++++++++------------
 1 file changed, 81 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/yandexdisk.py b/youtube_dl/extractor/yandexdisk.py
index e8f6ae10f..21f37c192 100644
--- a/youtube_dl/extractor/yandexdisk.py
+++ b/youtube_dl/extractor/yandexdisk.py
@@ -1,19 +1,40 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import json
+
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import compat_HTTPError
 from ..utils import (
     determine_ext,
+    ExtractorError,
     float_or_none,
     int_or_none,
-    try_get,
-    urlencode_postdata,
+    mimetype2ext,
+    parse_iso8601,
+    urljoin,
 )
 
 
 class YandexDiskIE(InfoExtractor):
-    _VALID_URL = r'https?://yadi\.sk/[di]/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'''(?x)https?://
+        (?:
+            (?:www\.)?yadi\.sk|
+            disk\.yandex\.
+                (?:
+                    az|
+                    by|
+                    co(?:m(?:\.(?:am|ge|tr))?|\.il)|
+                    ee|
+                    fr|
+                    k[gz]|
+                    l[tv]|
+                    md|
+                    t[jm]|
+                    u[az]|
+                    ru
+                )
+        )/(?:[di]/|public.*?\bhash=)(?P<id>[^/?#&]+)'''
 
     _TESTS = [{
         'url': 'https://yadi.sk/i/VdOeDou8eZs6Y',
@@ -25,94 +46,97 @@ class YandexDiskIE(InfoExtractor):
             'duration': 168.6,
             'uploader': 'y.botova',
             'uploader_id': '300043621',
+            'timestamp': 1421396809,
+            'upload_date': '20150116',
             'view_count': int,
         },
     }, {
         'url': 'https://yadi.sk/d/h3WAXvDS3Li3Ce',
         'only_matching': True,
+    }, {
+        'url': 'https://yadi.sk/public?hash=5DZ296JK9GWCLp02f6jrObjnctjRxMs8L6%2B%2FuhNqk38%3D',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        status = self._download_webpage(
-            'https://disk.yandex.com/auth/status', video_id, query={
-                'urlOrigin': url,
-                'source': 'public',
-                'md5': 'false',
-            })
+        try:
+            resource = self._download_json(
+                'https://cloud-api.yandex.net/v1/disk/public/resources',
+                video_id, query={'public_key': url})
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                error_description = self._parse_json(
+                    e.cause.read().decode(), video_id)['description']
+                raise ExtractorError(error_description, expected=True)
+            raise
 
-        sk = self._search_regex(
-            r'(["\'])sk(?:External)?\1\s*:\s*(["\'])(?P<value>(?:(?!\2).)+)\2',
-            status, 'sk', group='value')
+        title = resource['name']
+        public_url = resource.get('public_url')
+        if public_url:
+            video_id = self._match_id(public_url)
 
-        webpage = self._download_webpage(url, video_id)
+        self._set_cookie('yadi.sk', 'yandexuid', '0')
 
-        models = self._parse_json(
-            self._search_regex(
-                r'<script[^>]+id=["\']models-client[^>]+>\s*(\[.+?\])\s*</script',
-                webpage, 'video JSON'),
-            video_id)
-
-        data = next(
-            model['data'] for model in models
-            if model.get('model') == 'resource')
-
-        video_hash = data['id']
-        title = data['name']
-
-        models = self._download_json(
-            'https://disk.yandex.com/models/', video_id,
-            data=urlencode_postdata({
-                '_model.0': 'videoInfo',
-                'id.0': video_hash,
-                '_model.1': 'do-get-resource-url',
-                'id.1': video_hash,
-                'version': '13.6',
-                'sk': sk,
-            }), query={'_m': 'videoInfo'})['models']
-
-        videos = try_get(models, lambda x: x[0]['data']['videos'], list) or []
-        source_url = try_get(
-            models, lambda x: x[1]['data']['file'], compat_str)
+        def call_api(action):
+            return (self._download_json(
+                urljoin(url, '/public/api/') + action, video_id, data=json.dumps({
+                    'hash': url,
+                    # obtain sk if needed from call_api('check-auth') while
+                    # the yandexuid cookie is set and sending an empty JSON object
+                    'sk': 'ya6b52f8c6b12abe91a66d22d3a31084b'
+                }).encode(), headers={
+                    'Content-Type': 'text/plain',
+                }, fatal=False) or {}).get('data') or {}
 
         formats = []
+        source_url = resource.get('file')
+        if not source_url:
+            source_url = call_api('download-url').get('url')
         if source_url:
             formats.append({
                 'url': source_url,
                 'format_id': 'source',
-                'ext': determine_ext(title, 'mp4'),
+                'ext': determine_ext(title, mimetype2ext(resource.get('mime_type')) or 'mp4'),
                 'quality': 1,
+                'filesize': int_or_none(resource.get('size'))
             })
-        for video in videos:
+
+        video_streams = call_api('get-video-streams')
+        for video in (video_streams.get('videos') or []):
             format_url = video.get('url')
             if not format_url:
                 continue
-            if determine_ext(format_url) == 'm3u8':
+            if video.get('dimension') == 'adaptive':
                 formats.extend(self._extract_m3u8_formats(
-                    format_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    format_url, video_id, 'mp4', 'm3u8_native',
                     m3u8_id='hls', fatal=False))
             else:
+                size = video.get('size') or {}
+                height = int_or_none(size.get('height'))
+                format_id = 'hls'
+                if height:
+                    format_id += '-%dp' % height
                 formats.append({
+                    'ext': 'mp4',
+                    'format_id': format_id,
+                    'height': height,
+                    'protocol': 'm3u8_native',
                     'url': format_url,
+                    'width': int_or_none(size.get('width')),
                 })
         self._sort_formats(formats)
 
-        duration = float_or_none(try_get(
-            models, lambda x: x[0]['data']['duration']), 1000)
-        uploader = try_get(
-            data, lambda x: x['user']['display_name'], compat_str)
-        uploader_id = try_get(
-            data, lambda x: x['user']['uid'], compat_str)
-        view_count = int_or_none(try_get(
-            data, lambda x: x['meta']['views_counter']))
+        owner = resource.get('owner') or {}
 
         return {
             'id': video_id,
             'title': title,
-            'duration': duration,
-            'uploader': uploader,
-            'uploader_id': uploader_id,
-            'view_count': view_count,
+            'duration': float_or_none(video_streams.get('duration'), 1000),
+            'uploader': owner.get('display_name'),
+            'uploader_id': owner.get('uid'),
+            'view_count': int_or_none(resource.get('views_count')),
+            'timestamp': parse_iso8601(resource.get('created')),
             'formats': formats,
         }

From a6f75e6e89d2f2deba783cb92369747c1b44797d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 30 Dec 2020 16:45:53 +0100
Subject: [PATCH 031/860] [yandexdisk] extract info from webpage

the public API does not return metadata when download limit is reached
---
 youtube_dl/extractor/yandexdisk.py | 89 ++++++++++++++++--------------
 1 file changed, 47 insertions(+), 42 deletions(-)

diff --git a/youtube_dl/extractor/yandexdisk.py b/youtube_dl/extractor/yandexdisk.py
index 21f37c192..6fcd8ee7e 100644
--- a/youtube_dl/extractor/yandexdisk.py
+++ b/youtube_dl/extractor/yandexdisk.py
@@ -2,24 +2,23 @@
 from __future__ import unicode_literals
 
 import json
+import re
 
 from .common import InfoExtractor
-from ..compat import compat_HTTPError
 from ..utils import (
     determine_ext,
-    ExtractorError,
     float_or_none,
     int_or_none,
     mimetype2ext,
-    parse_iso8601,
+    try_get,
     urljoin,
 )
 
 
 class YandexDiskIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://
-        (?:
-            (?:www\.)?yadi\.sk|
+        (?P<domain>
+            yadi\.sk|
             disk\.yandex\.
                 (?:
                     az|
@@ -38,7 +37,7 @@ class YandexDiskIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'https://yadi.sk/i/VdOeDou8eZs6Y',
-        'md5': '33955d7ae052f15853dc41f35f17581c',
+        'md5': 'a4a8d52958c8fddcf9845935070402ae',
         'info_dict': {
             'id': 'VdOeDou8eZs6Y',
             'ext': 'mp4',
@@ -46,10 +45,9 @@ class YandexDiskIE(InfoExtractor):
             'duration': 168.6,
             'uploader': 'y.botova',
             'uploader_id': '300043621',
-            'timestamp': 1421396809,
-            'upload_date': '20150116',
             'view_count': int,
         },
+        'expected_warnings': ['Unable to download JSON metadata'],
     }, {
         'url': 'https://yadi.sk/d/h3WAXvDS3Li3Ce',
         'only_matching': True,
@@ -59,51 +57,58 @@ class YandexDiskIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        domain, video_id = re.match(self._VALID_URL, url).groups()
 
-        try:
-            resource = self._download_json(
-                'https://cloud-api.yandex.net/v1/disk/public/resources',
-                video_id, query={'public_key': url})
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                error_description = self._parse_json(
-                    e.cause.read().decode(), video_id)['description']
-                raise ExtractorError(error_description, expected=True)
-            raise
+        webpage = self._download_webpage(url, video_id)
+        store = self._parse_json(self._search_regex(
+            r'<script[^>]+id="store-prefetch"[^>]*>\s*({.+?})\s*</script>',
+            webpage, 'store'), video_id)
+        resource = store['resources'][store['rootResourceId']]
 
         title = resource['name']
-        public_url = resource.get('public_url')
+        meta = resource.get('meta') or {}
+
+        public_url = meta.get('short_url')
         if public_url:
             video_id = self._match_id(public_url)
 
-        self._set_cookie('yadi.sk', 'yandexuid', '0')
+        source_url = (self._download_json(
+            'https://cloud-api.yandex.net/v1/disk/public/resources/download',
+            video_id, query={'public_key': url}, fatal=False) or {}).get('href')
+        video_streams = resource.get('videoStreams') or {}
+        video_hash = resource.get('hash') or url
+        environment = store.get('environment') or {}
+        sk = environment.get('sk')
+        yandexuid = environment.get('yandexuid')
+        if sk and yandexuid and not (source_url and video_streams):
+            self._set_cookie(domain, 'yandexuid', yandexuid)
 
-        def call_api(action):
-            return (self._download_json(
-                urljoin(url, '/public/api/') + action, video_id, data=json.dumps({
-                    'hash': url,
-                    # obtain sk if needed from call_api('check-auth') while
-                    # the yandexuid cookie is set and sending an empty JSON object
-                    'sk': 'ya6b52f8c6b12abe91a66d22d3a31084b'
-                }).encode(), headers={
-                    'Content-Type': 'text/plain',
-                }, fatal=False) or {}).get('data') or {}
+            def call_api(action):
+                return (self._download_json(
+                    urljoin(url, '/public/api/') + action, video_id, data=json.dumps({
+                        'hash': video_hash,
+                        'sk': sk,
+                    }).encode(), headers={
+                        'Content-Type': 'text/plain',
+                    }, fatal=False) or {}).get('data') or {}
+            if not source_url:
+                # TODO: figure out how to detect if download limit has
+                # been reached and then avoid unnecessary source format
+                # extraction requests
+                source_url = call_api('download-url').get('url')
+            if not video_streams:
+                video_streams = call_api('get-video-streams')
 
         formats = []
-        source_url = resource.get('file')
-        if not source_url:
-            source_url = call_api('download-url').get('url')
         if source_url:
             formats.append({
                 'url': source_url,
                 'format_id': 'source',
-                'ext': determine_ext(title, mimetype2ext(resource.get('mime_type')) or 'mp4'),
+                'ext': determine_ext(title, meta.get('ext') or mimetype2ext(meta.get('mime_type')) or 'mp4'),
                 'quality': 1,
-                'filesize': int_or_none(resource.get('size'))
+                'filesize': int_or_none(meta.get('size'))
             })
 
-        video_streams = call_api('get-video-streams')
         for video in (video_streams.get('videos') or []):
             format_url = video.get('url')
             if not format_url:
@@ -128,15 +133,15 @@ class YandexDiskIE(InfoExtractor):
                 })
         self._sort_formats(formats)
 
-        owner = resource.get('owner') or {}
+        uid = resource.get('uid')
+        display_name = try_get(store, lambda x: x['users'][uid]['displayName'])
 
         return {
             'id': video_id,
             'title': title,
             'duration': float_or_none(video_streams.get('duration'), 1000),
-            'uploader': owner.get('display_name'),
-            'uploader_id': owner.get('uid'),
-            'view_count': int_or_none(resource.get('views_count')),
-            'timestamp': parse_iso8601(resource.get('created')),
+            'uploader': display_name,
+            'uploader_id': uid,
+            'view_count': int_or_none(meta.get('views_counter')),
             'formats': formats,
         }

From 2e21b06ea2f7a1ee5e038bf4274e5a74a4f52c2c Mon Sep 17 00:00:00 2001
From: nixxo <nixxo@protonmail.com>
Date: Wed, 30 Dec 2020 18:12:17 +0100
Subject: [PATCH 032/860] [vvvvid] add playlists support (#27574)

closes #18130
---
 youtube_dl/extractor/extractors.py |  5 ++-
 youtube_dl/extractor/vvvvid.py     | 65 +++++++++++++++++++++++++++---
 2 files changed, 63 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 51e6a463a..254206a08 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1425,7 +1425,10 @@ from .vshare import VShareIE
 from .medialaan import MedialaanIE
 from .vube import VubeIE
 from .vuclip import VuClipIE
-from .vvvvid import VVVVIDIE
+from .vvvvid import (
+    VVVVIDIE,
+    VVVVIDShowIE,
+)
 from .vyborymos import VyboryMosIE
 from .vzaar import VzaarIE
 from .wakanim import WakanimIE
diff --git a/youtube_dl/extractor/vvvvid.py b/youtube_dl/extractor/vvvvid.py
index 6906cd2ab..5b8ea3665 100644
--- a/youtube_dl/extractor/vvvvid.py
+++ b/youtube_dl/extractor/vvvvid.py
@@ -12,7 +12,8 @@ from ..utils import (
 
 
 class VVVVIDIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vvvvid\.it/(?:#!)?(?:show|anime|film|series)/(?P<show_id>\d+)/[^/]+/(?P<season_id>\d+)/(?P<id>[0-9]+)'
+    _VALID_URL_BASE = r'https?://(?:www\.)?vvvvid\.it/(?:#!)?(?:show|anime|film|series)/'
+    _VALID_URL = r'%s(?P<show_id>\d+)/[^/]+/(?P<season_id>\d+)/(?P<id>[0-9]+)' % _VALID_URL_BASE
     _TESTS = [{
         # video_type == 'video/vvvvid'
         'url': 'https://www.vvvvid.it/#!show/434/perche-dovrei-guardarlo-di-dario-moccia/437/489048/ping-pong',
@@ -45,20 +46,26 @@ class VVVVIDIE(InfoExtractor):
             'https://www.vvvvid.it/user/login',
             None, headers=self.geo_verification_headers())['data']['conn_id']
 
-    def _real_extract(self, url):
-        show_id, season_id, video_id = re.match(self._VALID_URL, url).groups()
+    def _download_info(self, show_id, path, video_id, fatal=True):
         response = self._download_json(
-            'https://www.vvvvid.it/vvvvid/ondemand/%s/season/%s' % (show_id, season_id),
+            'https://www.vvvvid.it/vvvvid/ondemand/%s%s' % (show_id, path),
             video_id, headers=self.geo_verification_headers(), query={
                 'conn_id': self._conn_id,
-            })
+            }, fatal=fatal)
         if response['result'] == 'error':
             raise ExtractorError('%s said: %s' % (
                 self.IE_NAME, response['message']), expected=True)
+        return response['data']
+
+    def _real_extract(self, url):
+        show_id, season_id, video_id = re.match(self._VALID_URL, url).groups()
+
+        response = self._download_info(
+            show_id, '/season/%s' % season_id, video_id)
 
         vid = int(video_id)
         video_data = list(filter(
-            lambda episode: episode.get('video_id') == vid, response['data']))[0]
+            lambda episode: episode.get('video_id') == vid, response))[0]
         formats = []
 
         # vvvvid embed_info decryption algorithm is reverse engineered from function $ds(h) at vvvvid.js
@@ -156,3 +163,49 @@ class VVVVIDIE(InfoExtractor):
             'view_count': int_or_none(video_data.get('views')),
             'like_count': int_or_none(video_data.get('video_likes')),
         }
+
+
+class VVVVIDShowIE(VVVVIDIE):
+    _VALID_URL = r'(?P<base_url>%s(?P<show_id>\d+)/(?P<show_title>[^/]+))/?(?:$|[\?&].*$)?$' % VVVVIDIE._VALID_URL_BASE
+    _TESTS = [{
+        'url': 'https://www.vvvvid.it/show/156/psyco-pass',
+        'info_dict': {
+            'id': '156',
+            'title': 'Psycho-Pass',
+            'description': 'md5:94d572c0bd85894b193b8aebc9a3a806',
+        },
+        'playlist_count': 46,
+    }]
+
+    def _real_extract(self, url):
+        base_url, show_id, show_title = re.match(self._VALID_URL, url).groups()
+
+        response = self._download_info(
+            show_id, '/seasons/', show_title)
+
+        show_infos = self._download_info(
+            show_id, '/info/', show_title, fatal=False)
+
+        entries = []
+        for season in response:
+            episodes = season.get('episodes') or []
+            for episode in episodes:
+                season_id = str_or_none(episode.get('season_id'))
+                video_id = str_or_none(episode.get('video_id'))
+                if not (season_id and video_id):
+                    continue
+
+                video_url = '/'.join([base_url, season_id, video_id])
+
+                entries.append({
+                    '_type': 'url_transparent',
+                    'ie_key': VVVVIDIE.ie_key(),
+                    'url': video_url,
+                    'title': episode.get('title'),
+                    'thumbnail': episode.get('thumbnail'),
+                    'description': episode.get('description'),
+                    'season_number': int_or_none(episode.get('season_number')),
+                    'episode_number': int_or_none(episode.get('number')),
+                })
+        return self.playlist_result(
+            entries, show_id, show_infos.get('title'), show_infos.get('description'))

From 7b0f04ed1f72fd4e4b5b3e935e08a912857fa8c4 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 30 Dec 2020 18:16:47 +0100
Subject: [PATCH 033/860] [vvvvid] imporove info extraction

---
 youtube_dl/extractor/vvvvid.py | 78 ++++++++++++++++++++++------------
 1 file changed, 52 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/vvvvid.py b/youtube_dl/extractor/vvvvid.py
index 5b8ea3665..014a67e53 100644
--- a/youtube_dl/extractor/vvvvid.py
+++ b/youtube_dl/extractor/vvvvid.py
@@ -22,6 +22,16 @@ class VVVVIDIE(InfoExtractor):
             'id': '489048',
             'ext': 'mp4',
             'title': 'Ping Pong',
+            'duration': 239,
+            'series': '"Perché dovrei guardarlo?" di Dario Moccia',
+            'season_id': '437',
+            'season_number': 1,
+            'episode': 'Ping Pong',
+            'episode_number': 1,
+            'episode_id': '3334',
+            'view_count': int,
+            'like_count': int,
+            'repost_count': int,
         },
         'params': {
             'skip_download': True,
@@ -38,6 +48,9 @@ class VVVVIDIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        'url': 'https://www.vvvvid.it/show/434/perche-dovrei-guardarlo-di-dario-moccia/437/489048',
+        'only_matching': True
     }]
     _conn_id = None
 
@@ -48,24 +61,34 @@ class VVVVIDIE(InfoExtractor):
 
     def _download_info(self, show_id, path, video_id, fatal=True):
         response = self._download_json(
-            'https://www.vvvvid.it/vvvvid/ondemand/%s%s' % (show_id, path),
+            'https://www.vvvvid.it/vvvvid/ondemand/%s/%s' % (show_id, path),
             video_id, headers=self.geo_verification_headers(), query={
                 'conn_id': self._conn_id,
             }, fatal=fatal)
-        if response['result'] == 'error':
+        if not (response or fatal):
+            return
+        if response.get('result') == 'error':
             raise ExtractorError('%s said: %s' % (
                 self.IE_NAME, response['message']), expected=True)
         return response['data']
 
+    def _extract_common_video_info(self, video_data):
+        return {
+            'thumbnail': video_data.get('thumbnail'),
+            'episode_number': int_or_none(video_data.get('number')),
+            'episode_id': str_or_none(video_data.get('id')),
+        }
+
     def _real_extract(self, url):
         show_id, season_id, video_id = re.match(self._VALID_URL, url).groups()
 
         response = self._download_info(
-            show_id, '/season/%s' % season_id, video_id)
+            show_id, 'season/%s' % season_id, video_id)
 
         vid = int(video_id)
         video_data = list(filter(
             lambda episode: episode.get('video_id') == vid, response))[0]
+        title = video_data['title']
         formats = []
 
         # vvvvid embed_info decryption algorithm is reverse engineered from function $ds(h) at vvvvid.js
@@ -148,25 +171,25 @@ class VVVVIDIE(InfoExtractor):
                     'http://sb.top-ix.org/videomg/_definst_/mp4:%s/playlist.m3u8' % embed_code, video_id))
         self._sort_formats(formats)
 
-        return {
+        info = self._extract_common_video_info(video_data)
+        info.update({
             'id': video_id,
-            'title': video_data['title'],
+            'title': title,
             'formats': formats,
-            'thumbnail': video_data.get('thumbnail'),
             'duration': int_or_none(video_data.get('length')),
             'series': video_data.get('show_title'),
             'season_id': season_id,
             'season_number': video_data.get('season_number'),
-            'episode_id': str_or_none(video_data.get('id')),
-            'episode_number': int_or_none(video_data.get('number')),
-            'episode_title': video_data['title'],
+            'episode': title,
             'view_count': int_or_none(video_data.get('views')),
             'like_count': int_or_none(video_data.get('video_likes')),
-        }
+            'repost_count': int_or_none(video_data.get('video_shares')),
+        })
+        return info
 
 
 class VVVVIDShowIE(VVVVIDIE):
-    _VALID_URL = r'(?P<base_url>%s(?P<show_id>\d+)/(?P<show_title>[^/]+))/?(?:$|[\?&].*$)?$' % VVVVIDIE._VALID_URL_BASE
+    _VALID_URL = r'(?P<base_url>%s(?P<id>\d+)(?:/(?P<show_title>[^/?&#]+))?)/?(?:[?#&]|$)' % VVVVIDIE._VALID_URL_BASE
     _TESTS = [{
         'url': 'https://www.vvvvid.it/show/156/psyco-pass',
         'info_dict': {
@@ -175,37 +198,40 @@ class VVVVIDShowIE(VVVVIDIE):
             'description': 'md5:94d572c0bd85894b193b8aebc9a3a806',
         },
         'playlist_count': 46,
+    }, {
+        'url': 'https://www.vvvvid.it/show/156',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         base_url, show_id, show_title = re.match(self._VALID_URL, url).groups()
 
-        response = self._download_info(
-            show_id, '/seasons/', show_title)
+        seasons = self._download_info(
+            show_id, 'seasons/', show_title)
 
-        show_infos = self._download_info(
-            show_id, '/info/', show_title, fatal=False)
+        show_info = self._download_info(
+            show_id, 'info/', show_title, fatal=False)
 
         entries = []
-        for season in response:
+        for season in (seasons or []):
+            season_number = int_or_none(season.get('number'))
             episodes = season.get('episodes') or []
             for episode in episodes:
                 season_id = str_or_none(episode.get('season_id'))
                 video_id = str_or_none(episode.get('video_id'))
                 if not (season_id and video_id):
                     continue
-
-                video_url = '/'.join([base_url, season_id, video_id])
-
-                entries.append({
-                    '_type': 'url_transparent',
+                info = self._extract_common_video_info(episode)
+                info.update({
+                    '_type': 'url',
                     'ie_key': VVVVIDIE.ie_key(),
-                    'url': video_url,
+                    'url': '/'.join([base_url, season_id, video_id]),
                     'title': episode.get('title'),
-                    'thumbnail': episode.get('thumbnail'),
                     'description': episode.get('description'),
-                    'season_number': int_or_none(episode.get('season_number')),
-                    'episode_number': int_or_none(episode.get('number')),
+                    'season_number': season_number,
+                    'season_id': season_id,
                 })
+                entries.append(info)
+
         return self.playlist_result(
-            entries, show_id, show_infos.get('title'), show_infos.get('description'))
+            entries, show_id, show_info.get('title'), show_info.get('description'))

From d1d0612160ed2d753cc57b30483776b7c1b03473 Mon Sep 17 00:00:00 2001
From: ozburo <hq@ozburo.com>
Date: Tue, 22 Dec 2020 06:12:14 -0600
Subject: [PATCH 034/860] [redditr] Extract all thumbnails

---
 youtube_dl/extractor/reddit.py | 15 ++++++++++++++-
 1 file changed, 14 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/reddit.py b/youtube_dl/extractor/reddit.py
index 3b2abb262..2d1a1fd99 100644
--- a/youtube_dl/extractor/reddit.py
+++ b/youtube_dl/extractor/reddit.py
@@ -9,6 +9,7 @@ from ..utils import (
     float_or_none,
     try_get,
     url_or_none,
+    unescapeHTML,
 )
 
 
@@ -118,11 +119,23 @@ class RedditRIE(InfoExtractor):
         else:
             age_limit = None
 
+        thumbnails = []
+        images = try_get(
+            data, lambda x: x['preview']['images'][0]['resolutions']) or []
+        for image in images:
+            url = url_or_none(unescapeHTML(image['url']))
+            if url is not None:
+                thumbnails.append({
+                    'url': url,
+                    'width': int_or_none(image['width']),
+                    'height': int_or_none(image['height']),
+                })
+
         return {
             '_type': 'url_transparent',
             'url': video_url,
             'title': data.get('title'),
-            'thumbnail': url_or_none(data.get('thumbnail')),
+            'thumbnails': thumbnails,
             'timestamp': float_or_none(data.get('created_utc')),
             'uploader': data.get('author'),
             'duration': int_or_none(try_get(

From 4046ffe1e16a65196e113cb12aa2d935f7f17c06 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Dec 2020 05:06:54 +0700
Subject: [PATCH 035/860] [redditr] Fix review issues and extract source
 thumbnail (closes #27503)

---
 youtube_dl/extractor/reddit.py | 36 ++++++++++++++++++++++------------
 1 file changed, 24 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/reddit.py b/youtube_dl/extractor/reddit.py
index 2d1a1fd99..222fa0172 100644
--- a/youtube_dl/extractor/reddit.py
+++ b/youtube_dl/extractor/reddit.py
@@ -8,8 +8,8 @@ from ..utils import (
     int_or_none,
     float_or_none,
     try_get,
-    url_or_none,
     unescapeHTML,
+    url_or_none,
 )
 
 
@@ -57,7 +57,8 @@ class RedditRIE(InfoExtractor):
             'id': 'zv89llsvexdz',
             'ext': 'mp4',
             'title': 'That small heart attack.',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.*\.(?:jpg|png)',
+            'thumbnails': 'count:4',
             'timestamp': 1501941939,
             'upload_date': '20170805',
             'uploader': 'Antw87',
@@ -120,16 +121,27 @@ class RedditRIE(InfoExtractor):
             age_limit = None
 
         thumbnails = []
-        images = try_get(
-            data, lambda x: x['preview']['images'][0]['resolutions']) or []
-        for image in images:
-            url = url_or_none(unescapeHTML(image['url']))
-            if url is not None:
-                thumbnails.append({
-                    'url': url,
-                    'width': int_or_none(image['width']),
-                    'height': int_or_none(image['height']),
-                })
+
+        def add_thumbnail(src):
+            if not isinstance(src, dict):
+                return
+            thumbnail_url = url_or_none(src.get('url'))
+            if not thumbnail_url:
+                return
+            thumbnails.append({
+                'url': unescapeHTML(thumbnail_url),
+                'width': int_or_none(src.get('width')),
+                'height': int_or_none(src.get('height')),
+            })
+
+        for image in try_get(data, lambda x: x['preview']['images']) or []:
+            if not isinstance(image, dict):
+                continue
+            add_thumbnail(image.get('source'))
+            resolutions = image.get('resolutions')
+            if isinstance(resolutions, list):
+                for resolution in resolutions:
+                    add_thumbnail(resolution)
 
         return {
             '_type': 'url_transparent',

From 2a84694b1e8d91b271eb5bb74b9d63cbfcc58cbe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Dec 2020 05:14:33 +0700
Subject: [PATCH 036/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 23 +++++++++++++++++++++++
 1 file changed, 23 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 2dad14949..1045fef5e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,26 @@
+version <unreleased>
+
+Core
+* [utils] Accept only supported protocols in url_or_none
+* [YoutubeDL] Allow format filtering using audio language (#16209)
+
+Extractors
++ [redditr] Extract all thumbnails (#27503)
+* [vvvvid] Improve info extraction
++ [vvvvid] Add support for playlists (#18130, #27574)
++ [yandexdisk] Extract info from webpage
+* [yandexdisk] Fix extraction (#17861, #27131)
+* [yandexvideo] Use old API call as fallback
+* [yandexvideo] Fix extraction (#25000)
+- [nbc] Remove CSNNE extractor
+* [nbc] Fix NBCSport VPlayer URL extraction (#16640)
++ [aenetworks] Add support for biography.com (#3863)
+* [uktvplay] Match new video URLs (#17909)
+* [sevenplay] Detect API errors
+* [tenplay] Fix format extraction (#26653)
+* [brightcove] Raise error for DRM protected videos (#23467, #27568)
+
+
 version 2020.12.29
 
 Extractors

From 4066945919a3f51a01f551ade8957f4856c67e65 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 31 Dec 2020 05:17:55 +0700
Subject: [PATCH 037/860] release 2020.12.31

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 docs/supportedsites.md                           | 4 +++-
 youtube_dl/version.py                            | 2 +-
 8 files changed, 17 insertions(+), 15 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 0d5a8e666..d1a6ad1f6 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.29. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2020.12.29**
+- [ ] I've verified that I'm running youtube-dl version **2020.12.31**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2020.12.29
+ [debug] youtube-dl version 2020.12.31
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index 88a6ea3fc..ded5beadf 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.29. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2020.12.29**
+- [ ] I've verified that I'm running youtube-dl version **2020.12.31**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 1d75d1a79..0e06de8dc 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.29. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2020.12.29**
+- [ ] I've verified that I'm running youtube-dl version **2020.12.31**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index 83f1f43cd..dfbfe3701 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.29. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2020.12.29**
+- [ ] I've verified that I'm running youtube-dl version **2020.12.31**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2020.12.29
+ [debug] youtube-dl version 2020.12.31
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index d3e03f78b..30061808d 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.29. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2020.12.29**
+- [ ] I've verified that I'm running youtube-dl version **2020.12.31**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index 1045fef5e..3782ad090 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2020.12.31
 
 Core
 * [utils] Accept only supported protocols in url_or_none
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 1406ba8b8..b1112f83b 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -104,6 +104,7 @@
  - **BilibiliAudioAlbum**
  - **BiliBiliPlayer**
  - **BioBioChileTV**
+ - **Biography**
  - **BIQLE**
  - **BitChute**
  - **BitChuteChannel**
@@ -197,7 +198,6 @@
  - **CrooksAndLiars**
  - **crunchyroll**
  - **crunchyroll:playlist**
- - **CSNNE**
  - **CSpan**: C-SPAN
  - **CtsNews**: 華視新聞
  - **CTV**
@@ -349,6 +349,7 @@
  - **hgtv.com:show**
  - **HiDive**
  - **HistoricFilms**
+ - **history:player**
  - **history:topic**: History.com Topic
  - **hitbox**
  - **hitbox:live**
@@ -1088,6 +1089,7 @@
  - **vube**: Vube.com
  - **VuClip**
  - **VVVVID**
+ - **VVVVIDShow**
  - **VyboryMos**
  - **Vzaar**
  - **Wakanim**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c7fb697c4..f795f0735 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2020.12.29'
+__version__ = '2020.12.31'

From 2f56caf0835786fa5af92da428ef90a530d90f26 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 31 Dec 2020 11:58:52 +0100
Subject: [PATCH 038/860] [yandexvideo] fix extraction for Python 3.4

---
 youtube_dl/extractor/yandexvideo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/yandexvideo.py b/youtube_dl/extractor/yandexvideo.py
index ab8c84c93..6a166ec9b 100644
--- a/youtube_dl/extractor/yandexvideo.py
+++ b/youtube_dl/extractor/yandexvideo.py
@@ -66,7 +66,7 @@ class YandexVideoIE(InfoExtractor):
         video_id = self._match_id(url)
 
         player = try_get((self._download_json(
-            'https://frontend.vh.yandex.ru/graphql', video_id, data=b'''{
+            'https://frontend.vh.yandex.ru/graphql', video_id, data=('''{
   player(content_id: "%s") {
     computed_title
     content_url
@@ -86,7 +86,7 @@ class YandexVideoIE(InfoExtractor):
     title
     views_count
   }
-}''' % video_id.encode(), fatal=False)), lambda x: x['player']['content'])
+}''' % video_id).encode(), fatal=False)), lambda x: x['player']['content'])
         if not player or player.get('error'):
             player = self._download_json(
                 'https://frontend.vh.yandex.ru/v23/player/%s.json' % video_id,

From c35bc8260665e071d5088f5be3bb37f7664ef06e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 31 Dec 2020 12:04:35 +0100
Subject: [PATCH 039/860] [vvvvid] skip unplayable episodes and extract akamai
 formats(closes #27599)

---
 youtube_dl/extractor/vvvvid.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/vvvvid.py b/youtube_dl/extractor/vvvvid.py
index 014a67e53..145805492 100644
--- a/youtube_dl/extractor/vvvvid.py
+++ b/youtube_dl/extractor/vvvvid.py
@@ -152,7 +152,6 @@ class VVVVIDIE(InfoExtractor):
             embed_code = ds(embed_code)
             video_type = video_data.get('video_type')
             if video_type in ('video/rcs', 'video/kenc'):
-                embed_code = re.sub(r'https?://([^/]+)/z/', r'https://\1/i/', embed_code).replace('/manifest.f4m', '/master.m3u8')
                 if video_type == 'video/kenc':
                     kenc = self._download_json(
                         'https://www.vvvvid.it/kenc', video_id, query={
@@ -163,9 +162,7 @@ class VVVVIDIE(InfoExtractor):
                     kenc_message = kenc.get('message')
                     if kenc_message:
                         embed_code += '?' + ds(kenc_message)
-                formats.extend(self._extract_m3u8_formats(
-                    embed_code, video_id, 'mp4',
-                    m3u8_id='hls', fatal=False))
+                formats.extend(self._extract_akamai_formats(embed_code, video_id))
             else:
                 formats.extend(self._extract_wowza_formats(
                     'http://sb.top-ix.org/videomg/_definst_/mp4:%s/playlist.m3u8' % embed_code, video_id))
@@ -217,6 +214,8 @@ class VVVVIDShowIE(VVVVIDIE):
             season_number = int_or_none(season.get('number'))
             episodes = season.get('episodes') or []
             for episode in episodes:
+                if episode.get('playable') is False:
+                    continue
                 season_id = str_or_none(episode.get('season_id'))
                 video_id = str_or_none(episode.get('video_id'))
                 if not (season_id and video_id):

From 4d7d056909665f68e2aaa95fe42a000c287265b6 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 31 Dec 2020 12:17:55 +0100
Subject: [PATCH 040/860] [sky] add support for Sports News articles and
 Brighcove videos(closes #13054)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/sky.py        | 99 +++++++++++++++++++++++-------
 2 files changed, 78 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 254206a08..54cbbe9d2 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1052,6 +1052,7 @@ from .skynewsarabia import (
 from .sky import (
     SkyNewsIE,
     SkySportsIE,
+    SkySportsNewsIE,
 )
 from .slideshare import SlideshareIE
 from .slideslive import SlidesLiveIE
diff --git a/youtube_dl/extractor/sky.py b/youtube_dl/extractor/sky.py
index 681691004..ff2c977a0 100644
--- a/youtube_dl/extractor/sky.py
+++ b/youtube_dl/extractor/sky.py
@@ -1,6 +1,8 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
 from ..utils import (
     extract_attributes,
@@ -11,36 +13,59 @@ from ..utils import (
 
 
 class SkyBaseIE(InfoExtractor):
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        video_data = extract_attributes(self._search_regex(
-            r'(<div.+?class="[^"]*sdc-article-video__media-ooyala[^"]*"[^>]+>)',
-            webpage, 'video data'))
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s'
+    _SDC_EL_REGEX = r'(?s)(<div[^>]+data-(?:component-name|fn)="sdc-(?:articl|sit)e-video"[^>]*>)'
 
-        video_url = 'ooyala:%s' % video_data['data-video-id']
-        if video_data.get('data-token-required') == 'true':
-            token_fetch_options = self._parse_json(video_data.get(
-                'data-token-fetch-options', '{}'), video_id, fatal=False) or {}
-            token_fetch_url = token_fetch_options.get('url')
-            if token_fetch_url:
-                embed_token = self._download_webpage(urljoin(
-                    url, token_fetch_url), video_id, fatal=False)
-                if embed_token:
-                    video_url = smuggle_url(
-                        video_url, {'embed_token': embed_token.strip('"')})
+    def _process_ooyala_element(self, webpage, sdc_el, url):
+        sdc = extract_attributes(sdc_el)
+        provider = sdc.get('data-provider')
+        if provider == 'ooyala':
+            video_id = sdc['data-sdc-video-id']
+            video_url = 'ooyala:%s' % video_id
+            ie_key = 'Ooyala'
+            ooyala_el = self._search_regex(
+                r'(<div[^>]+class="[^"]*\bsdc-article-video__media-ooyala\b[^"]*"[^>]+data-video-id="%s"[^>]*>)' % video_id,
+                webpage, 'video data', fatal=False)
+            if ooyala_el:
+                ooyala_attrs = extract_attributes(ooyala_el) or {}
+                if ooyala_attrs.get('data-token-required') == 'true':
+                    token_fetch_url = (self._parse_json(ooyala_attrs.get(
+                        'data-token-fetch-options', '{}'),
+                        video_id, fatal=False) or {}).get('url')
+                    if token_fetch_url:
+                        embed_token = self._download_json(urljoin(
+                            url, token_fetch_url), video_id, fatal=False)
+                        if embed_token:
+                            video_url = smuggle_url(
+                                video_url, {'embed_token': embed_token})
+        elif provider == 'brightcove':
+            video_id = sdc['data-video-id']
+            account_id = sdc.get('data-account-id') or '6058004172001'
+            player_id = sdc.get('data-player-id') or 'RC9PQUaJ6'
+            video_url = self.BRIGHTCOVE_URL_TEMPLATE % (account_id, player_id, video_id)
+            ie_key = 'BrightcoveNew'
 
         return {
             '_type': 'url_transparent',
             'id': video_id,
             'url': video_url,
+            'ie_key': ie_key,
+        }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        info = self._process_ooyala_element(webpage, self._search_regex(
+            self._SDC_EL_REGEX, webpage, 'sdc element'), url)
+        info.update({
             'title': self._og_search_title(webpage),
             'description': strip_or_none(self._og_search_description(webpage)),
-            'ie_key': 'Ooyala',
-        }
+        })
+        return info
 
 
 class SkySportsIE(SkyBaseIE):
+    IE_NAME = 'sky:sports'
     _VALID_URL = r'https?://(?:www\.)?skysports\.com/watch/video/([^/]+/)*(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://www.skysports.com/watch/video/10328419/bale-its-our-time-to-shine',
@@ -62,15 +87,45 @@ class SkySportsIE(SkyBaseIE):
 
 
 class SkyNewsIE(SkyBaseIE):
+    IE_NAME = 'sky:news'
     _VALID_URL = r'https?://news\.sky\.com/video/[0-9a-z-]+-(?P<id>[0-9]+)'
     _TEST = {
         'url': 'https://news.sky.com/video/russian-plane-inspected-after-deadly-fire-11712962',
-        'md5': 'd6327e581473cea9976a3236ded370cd',
+        'md5': '411e8893fd216c75eaf7e4c65d364115',
         'info_dict': {
-            'id': '1ua21xaDE6lCtZDmbYfl8kwsKLooJbNM',
+            'id': 'ref:1ua21xaDE6lCtZDmbYfl8kwsKLooJbNM',
             'ext': 'mp4',
             'title': 'Russian plane inspected after deadly fire',
             'description': 'The Russian Investigative Committee has released video of the wreckage of a passenger plane which caught fire near Moscow.',
+            'uploader_id': '6058004172001',
+            'timestamp': 1567112345,
+            'upload_date': '20190829',
         },
-        'add_ie': ['Ooyala'],
+        'add_ie': ['BrightcoveNew'],
     }
+
+
+class SkySportsNewsIE(SkyBaseIE):
+    IE_NAME = 'sky:sports:news'
+    _VALID_URL = r'https?://(?:www\.)?skysports\.com/([^/]+/)*news/\d+/(?P<id>\d+)'
+    _TEST = {
+        'url': 'http://www.skysports.com/golf/news/12176/10871916/dustin-johnson-ready-to-conquer-players-championship-at-tpc-sawgrass',
+        'info_dict': {
+            'id': '10871916',
+            'title': 'Dustin Johnson ready to conquer Players Championship at TPC Sawgrass',
+            'description': 'Dustin Johnson is confident he can continue his dominant form in 2017 by adding the Players Championship to his list of victories.',
+        },
+        'playlist_count': 2,
+    }
+
+    def _real_extract(self, url):
+        article_id = self._match_id(url)
+        webpage = self._download_webpage(url, article_id)
+
+        entries = []
+        for sdc_el in re.findall(self._SDC_EL_REGEX, webpage):
+            entries.append(self._process_ooyala_element(webpage, sdc_el, url))
+
+        return self.playlist_result(
+            entries, article_id, self._og_search_title(webpage),
+            self._html_search_meta(['og:description', 'description'], webpage))

From ab89a8678b77a53ef3ca701868a3acd5a4c300c1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 31 Dec 2020 23:31:07 +0100
Subject: [PATCH 041/860] [arcpublishing] Add new extractor

closes #2298
closes #9340
closes #17200
---
 youtube_dl/extractor/arcpublishing.py  | 173 +++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py     |   1 +
 youtube_dl/extractor/generic.py        |  19 +++
 youtube_dl/extractor/washingtonpost.py | 101 +++------------
 4 files changed, 210 insertions(+), 84 deletions(-)
 create mode 100644 youtube_dl/extractor/arcpublishing.py

diff --git a/youtube_dl/extractor/arcpublishing.py b/youtube_dl/extractor/arcpublishing.py
new file mode 100644
index 000000000..d1fb1a054
--- /dev/null
+++ b/youtube_dl/extractor/arcpublishing.py
@@ -0,0 +1,173 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    extract_attributes,
+    int_or_none,
+    parse_iso8601,
+    try_get,
+)
+
+
+class ArcPublishingIE(InfoExtractor):
+    _UUID_REGEX = r'[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12}'
+    _VALID_URL = r'arcpublishing:(?P<org>[a-z]+):(?P<id>%s)' % _UUID_REGEX
+    _TESTS = [{
+        # https://www.adn.com/politics/2020/11/02/video-senate-candidates-campaign-in-anchorage-on-eve-of-election-day/
+        'url': 'arcpublishing:adn:8c99cb6e-b29c-4bc9-9173-7bf9979225ab',
+        'only_matching': True,
+    }, {
+        # https://www.bostonglobe.com/video/2020/12/30/metro/footage-released-showing-officer-talking-about-striking-protesters-with-car/
+        'url': 'arcpublishing:bostonglobe:232b7ae6-7d73-432d-bc0a-85dbf0119ab1',
+        'only_matching': True,
+    }, {
+        # https://www.actionnewsjax.com/video/live-stream/
+        'url': 'arcpublishing:cmg:cfb1cf1b-3ab5-4d1b-86c5-a5515d311f2a',
+        'only_matching': True,
+    }, {
+        # https://elcomercio.pe/videos/deportes/deporte-total-futbol-peruano-seleccion-peruana-la-valorizacion-de-los-peruanos-en-el-exterior-tras-un-2020-atipico-nnav-vr-video-noticia/
+        'url': 'arcpublishing:elcomercio:27a7e1f8-2ec7-4177-874f-a4feed2885b3',
+        'only_matching': True,
+    }, {
+        # https://www.clickondetroit.com/video/community/2020/05/15/events-surrounding-woodward-dream-cruise-being-canceled/
+        'url': 'arcpublishing:gmg:c8793fb2-8d44-4242-881e-2db31da2d9fe',
+        'only_matching': True,
+    }, {
+        # https://www.wabi.tv/video/2020/12/30/trenton-company-making-equipment-pfizer-covid-vaccine/
+        'url': 'arcpublishing:gray:0b0ba30e-032a-4598-8810-901d70e6033e',
+        'only_matching': True,
+    }, {
+        # https://www.lateja.cr/el-mundo/video-china-aprueba-con-condiciones-su-primera/dfcbfa57-527f-45ff-a69b-35fe71054143/video/
+        'url': 'arcpublishing:gruponacion:dfcbfa57-527f-45ff-a69b-35fe71054143',
+        'only_matching': True,
+    }, {
+        # https://www.fifthdomain.com/video/2018/03/09/is-america-vulnerable-to-a-cyber-attack/
+        'url': 'arcpublishing:mco:aa0ca6fe-1127-46d4-b32c-be0d6fdb8055',
+        'only_matching': True,
+    }, {
+        # https://www.vl.no/kultur/2020/12/09/en-melding-fra-en-lytter-endret-julelista-til-lewi-bergrud/
+        'url': 'arcpublishing:mentormedier:47a12084-650b-4011-bfd0-3699b6947b2d',
+        'only_matching': True,
+    }, {
+        # https://www.14news.com/2020/12/30/whiskey-theft-caught-camera-henderson-liquor-store/
+        'url': 'arcpublishing:raycom:b89f61f8-79fa-4c09-8255-e64237119bf7',
+        'only_matching': True,
+    }, {
+        # https://www.theglobeandmail.com/world/video-ethiopian-woman-who-became-symbol-of-integration-in-italy-killed-on/
+        'url': 'arcpublishing:tgam:411b34c1-8701-4036-9831-26964711664b',
+        'only_matching': True,
+    }, {
+        # https://www.pilotonline.com/460f2931-8130-4719-8ea1-ffcb2d7cb685-132.html
+        'url': 'arcpublishing:tronc:460f2931-8130-4719-8ea1-ffcb2d7cb685',
+        'only_matching': True,
+    }]
+    _POWA_DEFAULTS = [
+        (['cmg', 'prisa'], '%s-config-prod.api.cdn.arcpublishing.com/video'),
+        ([
+            'adn', 'advancelocal', 'answers', 'bonnier', 'bostonglobe', 'demo',
+            'gmg', 'gruponacion', 'infobae', 'mco', 'nzme', 'pmn', 'raycom',
+            'spectator', 'tbt', 'tgam', 'tronc', 'wapo', 'wweek',
+        ], 'video-api-cdn.%s.arcpublishing.com/api'),
+    ]
+
+    def _extract_urls(webpage):
+        entries = []
+        # https://arcpublishing.atlassian.net/wiki/spaces/POWA/overview
+        for powa_el in re.findall(r'(<div[^>]+class="[^"]*\bpowa\b[^"]*"[^>]+data-uuid="%s"[^>]*>)' % ArcPublishingIE._UUID_REGEX, webpage):
+            powa = extract_attributes(powa_el) or {}
+            org = powa.get('data-org')
+            uuid = powa.get('data-uuid')
+            if org and uuid:
+                entries.append('arcpublishing:%s:%s' % (org, uuid))
+        return entries
+
+    def _real_extract(self, url):
+        org, uuid = re.match(self._VALID_URL, url).groups()
+        for orgs, tmpl in self._POWA_DEFAULTS:
+            if org in orgs:
+                base_api_tmpl = tmpl
+                break
+        else:
+            base_api_tmpl = '%s-prod-cdn.video-api.arcpublishing.com/api'
+        if org == 'wapo':
+            org = 'washpost'
+        video = self._download_json(
+            'https://%s/v1/ansvideos/findByUuid' % (base_api_tmpl % org),
+            uuid, query={'uuid': uuid})[0]
+        title = video['headlines']['basic']
+        is_live = video.get('status') == 'live'
+
+        urls = []
+        formats = []
+        for s in video.get('streams', []):
+            s_url = s.get('url')
+            if not s_url or s_url in urls:
+                continue
+            urls.append(s_url)
+            stream_type = s.get('stream_type')
+            if stream_type == 'smil':
+                smil_formats = self._extract_smil_formats(
+                    s_url, uuid, fatal=False)
+                for f in smil_formats:
+                    if f['url'].endswith('/cfx/st'):
+                        f['app'] = 'cfx/st'
+                        if not f['play_path'].startswith('mp4:'):
+                            f['play_path'] = 'mp4:' + f['play_path']
+                        if isinstance(f['tbr'], float):
+                            f['vbr'] = f['tbr'] * 1000
+                            del f['tbr']
+                            f['format_id'] = 'rtmp-%d' % f['vbr']
+                formats.extend(smil_formats)
+            elif stream_type in ('ts', 'hls'):
+                m3u8_formats = self._extract_m3u8_formats(
+                    s_url, uuid, 'mp4', 'm3u8' if is_live else 'm3u8_native',
+                    m3u8_id='hls', fatal=False)
+                if all([f.get('acodec') == 'none' for f in m3u8_formats]):
+                    continue
+                for f in m3u8_formats:
+                    if f.get('acodec') == 'none':
+                        f['preference'] = -40
+                    elif f.get('vcodec') == 'none':
+                        f['preference'] = -50
+                    height = f.get('height')
+                    if not height:
+                        continue
+                    vbr = self._search_regex(
+                        r'[_x]%d[_-](\d+)' % height, f['url'], 'vbr', default=None)
+                    if vbr:
+                        f['vbr'] = int(vbr)
+                formats.extend(m3u8_formats)
+            else:
+                vbr = int_or_none(s.get('bitrate'))
+                formats.append({
+                    'format_id': '%s-%d' % (stream_type, vbr) if vbr else stream_type,
+                    'vbr': vbr,
+                    'width': int_or_none(s.get('width')),
+                    'height': int_or_none(s.get('height')),
+                    'filesize': int_or_none(s.get('filesize')),
+                    'url': s_url,
+                    'preference': -1,
+                })
+        self._sort_formats(
+            formats, ('preference', 'width', 'height', 'vbr', 'filesize', 'tbr', 'ext', 'format_id'))
+
+        subtitles = {}
+        for subtitle in (try_get(video, lambda x: x['subtitles']['urls'], list) or []):
+            subtitle_url = subtitle.get('url')
+            if subtitle_url:
+                subtitles.setdefault('en', []).append({'url': subtitle_url})
+
+        return {
+            'id': uuid,
+            'title': self._live_title(title) if is_live else title,
+            'thumbnail': try_get(video, lambda x: x['promo_image']['url']),
+            'description': try_get(video, lambda x: x['subheadlines']['basic']),
+            'formats': formats,
+            'duration': int_or_none(video.get('duration'), 100),
+            'timestamp': parse_iso8601(video.get('created_date')),
+            'subtitles': subtitles,
+            'is_live': is_live,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 54cbbe9d2..d9d3f4940 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -56,6 +56,7 @@ from .appletrailers import (
     AppleTrailersSectionIE,
 )
 from .archiveorg import ArchiveOrgIE
+from .arcpublishing import ArcPublishingIE
 from .arkena import ArkenaIE
 from .ard import (
     ARDBetaMediathekIE,
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 14c27c6da..780971a92 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -127,6 +127,7 @@ from .expressen import ExpressenIE
 from .zype import ZypeIE
 from .odnoklassniki import OdnoklassnikiIE
 from .kinja import KinjaEmbedIE
+from .arcpublishing import ArcPublishingIE
 
 
 class GenericIE(InfoExtractor):
@@ -2208,6 +2209,20 @@ class GenericIE(InfoExtractor):
                 'uploader': 'OTT Videos',
             },
         },
+        {
+            # ArcPublishing PoWa video player
+            'url': 'https://www.adn.com/politics/2020/11/02/video-senate-candidates-campaign-in-anchorage-on-eve-of-election-day/',
+            'md5': 'b03b2fac8680e1e5a7cc81a5c27e71b3',
+            'info_dict': {
+                'id': '8c99cb6e-b29c-4bc9-9173-7bf9979225ab',
+                'ext': 'mp4',
+                'title': 'Senate candidates wave to voters on Anchorage streets',
+                'description': 'md5:91f51a6511f090617353dc720318b20e',
+                'timestamp': 1604378735,
+                'upload_date': '20201103',
+                'duration': 1581,
+            },
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -2574,6 +2589,10 @@ class GenericIE(InfoExtractor):
         if tp_urls:
             return self.playlist_from_matches(tp_urls, video_id, video_title, ie='ThePlatform')
 
+        arc_urls = ArcPublishingIE._extract_urls(webpage)
+        if arc_urls:
+            return self.playlist_from_matches(arc_urls, video_id, video_title, ie=ArcPublishingIE.ie_key())
+
         # Look for embedded rtl.nl player
         matches = re.findall(
             r'<iframe[^>]+?src="((?:https?:)?//(?:(?:www|static)\.)?rtl\.nl/(?:system/videoplayer/[^"]+(?:video_)?)?embed[^"]+)"',
diff --git a/youtube_dl/extractor/washingtonpost.py b/youtube_dl/extractor/washingtonpost.py
index 625d0a1cc..8afb1af83 100644
--- a/youtube_dl/extractor/washingtonpost.py
+++ b/youtube_dl/extractor/washingtonpost.py
@@ -4,17 +4,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    strip_jsonp,
-)
 
 
 class WashingtonPostIE(InfoExtractor):
     IE_NAME = 'washingtonpost'
-    _VALID_URL = r'(?:washingtonpost:|https?://(?:www\.)?washingtonpost\.com/video/(?:[^/]+/)*)(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
+    _VALID_URL = r'(?:washingtonpost:|https?://(?:www\.)?washingtonpost\.com/(?:video|posttv)/(?:[^/]+/)*)(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
     _EMBED_URL = r'https?://(?:www\.)?washingtonpost\.com/video/c/embed/[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12}'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://www.washingtonpost.com/video/c/video/480ba4ee-1ec7-11e6-82c2-a7dcb313287d',
         'md5': '6f537e1334b714eb15f9563bd4b9cdfa',
         'info_dict': {
@@ -23,10 +19,15 @@ class WashingtonPostIE(InfoExtractor):
             'title': 'Egypt finds belongings, debris from plane crash',
             'description': 'md5:a17ceee432f215a5371388c1f680bd86',
             'upload_date': '20160520',
-            'uploader': 'Reuters',
-            'timestamp': 1463778452,
+            'timestamp': 1463775187,
         },
-    }
+    }, {
+        'url': 'https://www.washingtonpost.com/video/world/egypt-finds-belongings-debris-from-plane-crash/2016/05/20/480ba4ee-1ec7-11e6-82c2-a7dcb313287d_video.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.washingtonpost.com/posttv/world/iraq-to-track-down-antiquities-after-islamic-state-museum-rampage/2015/02/28/7c57e916-bf86-11e4-9dfb-03366e719af8_video.html',
+        'only_matching': True,
+    }]
 
     @classmethod
     def _extract_urls(cls, webpage):
@@ -35,73 +36,8 @@ class WashingtonPostIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        video_data = self._download_json(
-            'http://www.washingtonpost.com/posttv/c/videojson/%s?resType=jsonp' % video_id,
-            video_id, transform_source=strip_jsonp)[0]['contentConfig']
-        title = video_data['title']
-
-        urls = []
-        formats = []
-        for s in video_data.get('streams', []):
-            s_url = s.get('url')
-            if not s_url or s_url in urls:
-                continue
-            urls.append(s_url)
-            video_type = s.get('type')
-            if video_type == 'smil':
-                continue
-            elif video_type in ('ts', 'hls') and ('_master.m3u8' in s_url or '_mobile.m3u8' in s_url):
-                m3u8_formats = self._extract_m3u8_formats(
-                    s_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls', fatal=False)
-                for m3u8_format in m3u8_formats:
-                    width = m3u8_format.get('width')
-                    if not width:
-                        continue
-                    vbr = self._search_regex(
-                        r'%d_%d_(\d+)' % (width, m3u8_format['height']), m3u8_format['url'], 'vbr', default=None)
-                    if vbr:
-                        m3u8_format.update({
-                            'vbr': int_or_none(vbr),
-                        })
-                formats.extend(m3u8_formats)
-            else:
-                width = int_or_none(s.get('width'))
-                vbr = int_or_none(s.get('bitrate'))
-                has_width = width != 0
-                formats.append({
-                    'format_id': (
-                        '%s-%d-%d' % (video_type, width, vbr)
-                        if width
-                        else video_type),
-                    'vbr': vbr if has_width else None,
-                    'width': width,
-                    'height': int_or_none(s.get('height')),
-                    'acodec': s.get('audioCodec'),
-                    'vcodec': s.get('videoCodec') if has_width else 'none',
-                    'filesize': int_or_none(s.get('fileSize')),
-                    'url': s_url,
-                    'ext': 'mp4',
-                    'protocol': 'm3u8_native' if video_type in ('ts', 'hls') else None,
-                })
-        source_media_url = video_data.get('sourceMediaURL')
-        if source_media_url:
-            formats.append({
-                'format_id': 'source_media',
-                'url': source_media_url,
-            })
-        self._sort_formats(
-            formats, ('width', 'height', 'vbr', 'filesize', 'tbr', 'format_id'))
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': video_data.get('blurb'),
-            'uploader': video_data.get('credits', {}).get('source'),
-            'formats': formats,
-            'duration': int_or_none(video_data.get('videoDuration'), 100),
-            'timestamp': int_or_none(
-                video_data.get('dateConfig', {}).get('dateFirstPublished'), 1000),
-        }
+        return self.url_result(
+            'arcpublishing:wapo:' + video_id, 'ArcPublishing', video_id)
 
 
 class WashingtonPostArticleIE(InfoExtractor):
@@ -121,9 +57,8 @@ class WashingtonPostArticleIE(InfoExtractor):
                 'title': 'Breaking Points: The Paper Mine',
                 'duration': 1290,
                 'description': 'Overly complicated paper pushing is nothing new to government bureaucracy. But the way federal retirement applications are filed may be the most outdated. David Fahrenthold explains.',
-                'uploader': 'The Washington Post',
-                'timestamp': 1395527908,
-                'upload_date': '20140322',
+                'timestamp': 1395440416,
+                'upload_date': '20140321',
             },
         }, {
             'md5': '1fff6a689d8770966df78c8cb6c8c17c',
@@ -133,9 +68,8 @@ class WashingtonPostArticleIE(InfoExtractor):
                 'title': 'The town bureaucracy sustains',
                 'description': 'Underneath the friendly town of Boyers is a sea of government paperwork. In a disused limestone mine, hundreds of locals now track, file and process retirement applications for the federal government. We set out to find out what it\'s like to do paperwork 230 feet underground.',
                 'duration': 2220,
-                'timestamp': 1395528005,
-                'upload_date': '20140322',
-                'uploader': 'The Washington Post',
+                'timestamp': 1395441819,
+                'upload_date': '20140321',
             },
         }],
     }, {
@@ -151,8 +85,7 @@ class WashingtonPostArticleIE(InfoExtractor):
                 'ext': 'mp4',
                 'description': 'Washington Post transportation reporter Ashley Halsey III explains why a plane\'s black box needs to be recovered from a crash site instead of having its information streamed in real time throughout the flight.',
                 'upload_date': '20141230',
-                'uploader': 'The Washington Post',
-                'timestamp': 1419974765,
+                'timestamp': 1419972442,
                 'title': 'Why black boxes don’t transmit data in real time',
             }
         }]

From 40d66e07dfef72fbef761ac720b82eb77deb7398 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 31 Dec 2020 23:42:17 +0100
Subject: [PATCH 042/860] [arcpublishing] add missing staticmethod decorator

---
 youtube_dl/extractor/arcpublishing.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/arcpublishing.py b/youtube_dl/extractor/arcpublishing.py
index d1fb1a054..ca6a6c4d8 100644
--- a/youtube_dl/extractor/arcpublishing.py
+++ b/youtube_dl/extractor/arcpublishing.py
@@ -73,6 +73,7 @@ class ArcPublishingIE(InfoExtractor):
         ], 'video-api-cdn.%s.arcpublishing.com/api'),
     ]
 
+    @staticmethod
     def _extract_urls(webpage):
         entries = []
         # https://arcpublishing.atlassian.net/wiki/spaces/POWA/overview

From 8c17afc4710cb39801bdae5027fe3bf641bf485d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 1 Jan 2021 17:05:42 +0100
Subject: [PATCH 043/860] [acast] fix extraction(closes #21444)(closes
 #27612)(closes #27613)

---
 youtube_dl/extractor/acast.py | 116 ++++++++++++++++------------------
 1 file changed, 53 insertions(+), 63 deletions(-)

diff --git a/youtube_dl/extractor/acast.py b/youtube_dl/extractor/acast.py
index b17c792d2..60378db1b 100644
--- a/youtube_dl/extractor/acast.py
+++ b/youtube_dl/extractor/acast.py
@@ -2,21 +2,47 @@
 from __future__ import unicode_literals
 
 import re
-import functools
 
 from .common import InfoExtractor
-from ..compat import compat_str
 from ..utils import (
     clean_html,
-    float_or_none,
     int_or_none,
-    try_get,
-    unified_timestamp,
-    OnDemandPagedList,
+    parse_iso8601,
 )
 
 
-class ACastIE(InfoExtractor):
+class ACastBaseIE(InfoExtractor):
+    def _extract_episode(self, episode, show_info):
+        title = episode['title']
+        info = {
+            'id': episode['id'],
+            'display_id': episode.get('episodeUrl'),
+            'url': episode['url'],
+            'title': title,
+            'description': clean_html(episode.get('description') or episode.get('summary')),
+            'thumbnail': episode.get('image'),
+            'timestamp': parse_iso8601(episode.get('publishDate')),
+            'duration': int_or_none(episode.get('duration')),
+            'filesize': int_or_none(episode.get('contentLength')),
+            'season_number': int_or_none(episode.get('season')),
+            'episode': title,
+            'episode_number': int_or_none(episode.get('episode')),
+        }
+        info.update(show_info)
+        return info
+
+    def _extract_show_info(self, show):
+        return {
+            'creator': show.get('author'),
+            'series': show.get('title'),
+        }
+
+    def _call_api(self, path, video_id, query=None):
+        return self._download_json(
+            'https://feeder.acast.com/api/v1/shows/' + path, video_id, query=query)
+
+
+class ACastIE(ACastBaseIE):
     IE_NAME = 'acast'
     _VALID_URL = r'''(?x)
                     https?://
@@ -28,15 +54,15 @@ class ACastIE(InfoExtractor):
                     '''
     _TESTS = [{
         'url': 'https://www.acast.com/sparpodcast/2.raggarmordet-rosterurdetforflutna',
-        'md5': '16d936099ec5ca2d5869e3a813ee8dc4',
+        'md5': 'f5598f3ad1e4776fed12ec1407153e4b',
         'info_dict': {
             'id': '2a92b283-1a75-4ad8-8396-499c641de0d9',
             'ext': 'mp3',
             'title': '2. Raggarmordet - Röster ur det förflutna',
-            'description': 'md5:4f81f6d8cf2e12ee21a321d8bca32db4',
+            'description': 'md5:a992ae67f4d98f1c0141598f7bebbf67',
             'timestamp': 1477346700,
             'upload_date': '20161024',
-            'duration': 2766.602563,
+            'duration': 2766,
             'creator': 'Anton Berg & Martin Johnson',
             'series': 'Spår',
             'episode': '2. Raggarmordet - Röster ur det förflutna',
@@ -45,7 +71,7 @@ class ACastIE(InfoExtractor):
         'url': 'http://embed.acast.com/adambuxton/ep.12-adam-joeschristmaspodcast2015',
         'only_matching': True,
     }, {
-        'url': 'https://play.acast.com/s/rattegangspodden/s04e09-styckmordet-i-helenelund-del-22',
+        'url': 'https://play.acast.com/s/rattegangspodden/s04e09styckmordetihelenelund-del2-2',
         'only_matching': True,
     }, {
         'url': 'https://play.acast.com/s/sparpodcast/2a92b283-1a75-4ad8-8396-499c641de0d9',
@@ -54,40 +80,14 @@ class ACastIE(InfoExtractor):
 
     def _real_extract(self, url):
         channel, display_id = re.match(self._VALID_URL, url).groups()
-        s = self._download_json(
-            'https://feeder.acast.com/api/v1/shows/%s/episodes/%s' % (channel, display_id),
-            display_id)
-        media_url = s['url']
-        if re.search(r'[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12}', display_id):
-            episode_url = s.get('episodeUrl')
-            if episode_url:
-                display_id = episode_url
-            else:
-                channel, display_id = re.match(self._VALID_URL, s['link']).groups()
-        cast_data = self._download_json(
-            'https://play-api.acast.com/splash/%s/%s' % (channel, display_id),
-            display_id)['result']
-        e = cast_data['episode']
-        title = e.get('name') or s['title']
-        return {
-            'id': compat_str(e['id']),
-            'display_id': display_id,
-            'url': media_url,
-            'title': title,
-            'description': e.get('summary') or clean_html(e.get('description') or s.get('description')),
-            'thumbnail': e.get('image'),
-            'timestamp': unified_timestamp(e.get('publishingDate') or s.get('publishDate')),
-            'duration': float_or_none(e.get('duration') or s.get('duration')),
-            'filesize': int_or_none(e.get('contentLength')),
-            'creator': try_get(cast_data, lambda x: x['show']['author'], compat_str),
-            'series': try_get(cast_data, lambda x: x['show']['name'], compat_str),
-            'season_number': int_or_none(e.get('seasonNumber')),
-            'episode': title,
-            'episode_number': int_or_none(e.get('episodeNumber')),
-        }
+        episode = self._call_api(
+            '%s/episodes/%s' % (channel, display_id),
+            display_id, {'showInfo': 'true'})
+        return self._extract_episode(
+            episode, self._extract_show_info(episode.get('show') or {}))
 
 
-class ACastChannelIE(InfoExtractor):
+class ACastChannelIE(ACastBaseIE):
     IE_NAME = 'acast:channel'
     _VALID_URL = r'''(?x)
                     https?://
@@ -102,34 +102,24 @@ class ACastChannelIE(InfoExtractor):
         'info_dict': {
             'id': '4efc5294-5385-4847-98bd-519799ce5786',
             'title': 'Today in Focus',
-            'description': 'md5:9ba5564de5ce897faeb12963f4537a64',
+            'description': 'md5:c09ce28c91002ce4ffce71d6504abaae',
         },
-        'playlist_mincount': 35,
+        'playlist_mincount': 200,
     }, {
         'url': 'http://play.acast.com/s/ft-banking-weekly',
         'only_matching': True,
     }]
-    _API_BASE_URL = 'https://play.acast.com/api/'
-    _PAGE_SIZE = 10
 
     @classmethod
     def suitable(cls, url):
         return False if ACastIE.suitable(url) else super(ACastChannelIE, cls).suitable(url)
 
-    def _fetch_page(self, channel_slug, page):
-        casts = self._download_json(
-            self._API_BASE_URL + 'channels/%s/acasts?page=%s' % (channel_slug, page),
-            channel_slug, note='Download page %d of channel data' % page)
-        for cast in casts:
-            yield self.url_result(
-                'https://play.acast.com/s/%s/%s' % (channel_slug, cast['url']),
-                'ACast', cast['id'])
-
     def _real_extract(self, url):
-        channel_slug = self._match_id(url)
-        channel_data = self._download_json(
-            self._API_BASE_URL + 'channels/%s' % channel_slug, channel_slug)
-        entries = OnDemandPagedList(functools.partial(
-            self._fetch_page, channel_slug), self._PAGE_SIZE)
-        return self.playlist_result(entries, compat_str(
-            channel_data['id']), channel_data['name'], channel_data.get('description'))
+        show_slug = self._match_id(url)
+        show = self._call_api(show_slug, show_slug)
+        show_info = self._extract_show_info(show)
+        entries = []
+        for episode in (show.get('episodes') or []):
+            entries.append(self._extract_episode(episode, show_info))
+        return self.playlist_result(
+            entries, show.get('id'), show.get('title'), show.get('description'))

From d0d838638c8b49514c8bf093d6c76fea98019971 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 1 Jan 2021 18:30:56 +0100
Subject: [PATCH 044/860] [stitcher] fix extraction(closes #20811)(closes
 #27606)

---
 youtube_dl/extractor/stitcher.py | 60 ++++++++++++++++++--------------
 1 file changed, 33 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/stitcher.py b/youtube_dl/extractor/stitcher.py
index 97d1ff681..b8b5711b1 100644
--- a/youtube_dl/extractor/stitcher.py
+++ b/youtube_dl/extractor/stitcher.py
@@ -4,25 +4,28 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
+    clean_html,
+    ExtractorError,
     int_or_none,
-    js_to_json,
-    unescapeHTML,
+    str_or_none,
+    try_get,
 )
 
 
 class StitcherIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?stitcher\.com/podcast/(?:[^/]+/)+e/(?:(?P<display_id>[^/#?&]+?)-)?(?P<id>\d+)(?:[/#?&]|$)'
+    _VALID_URL = r'https?://(?:www\.)?stitcher\.com/(?:podcast|show)/(?:[^/]+/)+e(?:pisode)?/(?:(?P<display_id>[^/#?&]+?)-)?(?P<id>\d+)(?:[/#?&]|$)'
     _TESTS = [{
         'url': 'http://www.stitcher.com/podcast/the-talking-machines/e/40789481?autoplay=true',
-        'md5': '391dd4e021e6edeb7b8e68fbf2e9e940',
+        'md5': 'e9635098e0da10b21a0e2b85585530f6',
         'info_dict': {
             'id': '40789481',
             'ext': 'mp3',
             'title': 'Machine Learning Mastery and Cancer Clusters',
-            'description': 'md5:55163197a44e915a14a1ac3a1de0f2d3',
+            'description': 'md5:547adb4081864be114ae3831b4c2b42f',
             'duration': 1604,
             'thumbnail': r're:^https?://.*\.jpg',
+            'upload_date': '20180126',
+            'timestamp': 1516989316,
         },
     }, {
         'url': 'http://www.stitcher.com/podcast/panoply/vulture-tv/e/the-rare-hourlong-comedy-plus-40846275?autoplay=true',
@@ -38,6 +41,7 @@ class StitcherIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+        'skip': 'Page Not Found',
     }, {
         # escaped title
         'url': 'http://www.stitcher.com/podcast/marketplace-on-stitcher/e/40910226?autoplay=true',
@@ -45,37 +49,39 @@ class StitcherIE(InfoExtractor):
     }, {
         'url': 'http://www.stitcher.com/podcast/panoply/getting-in/e/episode-2a-how-many-extracurriculars-should-i-have-40876278?autoplay=true',
         'only_matching': True,
+    }, {
+        'url': 'https://www.stitcher.com/show/threedom/episode/circles-on-a-stick-200212584',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        audio_id = mobj.group('id')
-        display_id = mobj.group('display_id') or audio_id
+        display_id, audio_id = re.match(self._VALID_URL, url).groups()
 
-        webpage = self._download_webpage(url, display_id)
+        resp = self._download_json(
+            'https://api.prod.stitcher.com/episode/' + audio_id,
+            display_id or audio_id)
+        episode = try_get(resp, lambda x: x['data']['episodes'][0], dict)
+        if not episode:
+            raise ExtractorError(resp['errors'][0]['message'], expected=True)
 
-        episode = self._parse_json(
-            js_to_json(self._search_regex(
-                r'(?s)var\s+stitcher(?:Config)?\s*=\s*({.+?});\n', webpage, 'episode config')),
-            display_id)['config']['episode']
+        title = episode['title'].strip()
+        audio_url = episode['audio_url']
 
-        title = unescapeHTML(episode['title'])
-        formats = [{
-            'url': episode[episode_key],
-            'ext': determine_ext(episode[episode_key]) or 'mp3',
-            'vcodec': 'none',
-        } for episode_key in ('episodeURL',) if episode.get(episode_key)]
-        description = self._search_regex(
-            r'Episode Info:\s*</span>([^<]+)<', webpage, 'description', fatal=False)
-        duration = int_or_none(episode.get('duration'))
-        thumbnail = episode.get('episodeImage')
+        thumbnail = None
+        show_id = episode.get('show_id')
+        if show_id and episode.get('classic_id') != -1:
+            thumbnail = 'https://stitcher-classic.imgix.net/feedimages/%s.jpg' % show_id
 
         return {
             'id': audio_id,
             'display_id': display_id,
             'title': title,
-            'description': description,
-            'duration': duration,
+            'description': clean_html(episode.get('html_description') or episode.get('description')),
+            'duration': int_or_none(episode.get('duration')),
             'thumbnail': thumbnail,
-            'formats': formats,
+            'url': audio_url,
+            'vcodec': 'none',
+            'timestamp': int_or_none(episode.get('date_created')),
+            'season_number': int_or_none(episode.get('season')),
+            'season_id': str_or_none(episode.get('season_id')),
         }

From 75972e200d033429bf9d34793ad3ffc813042347 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 1 Jan 2021 20:21:59 +0100
Subject: [PATCH 045/860] [vvvvid] fix season metadata extraction(#18130)

---
 youtube_dl/extractor/vvvvid.py | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/vvvvid.py b/youtube_dl/extractor/vvvvid.py
index 145805492..f4cae7fe9 100644
--- a/youtube_dl/extractor/vvvvid.py
+++ b/youtube_dl/extractor/vvvvid.py
@@ -25,7 +25,6 @@ class VVVVIDIE(InfoExtractor):
             'duration': 239,
             'series': '"Perché dovrei guardarlo?" di Dario Moccia',
             'season_id': '437',
-            'season_number': 1,
             'episode': 'Ping Pong',
             'episode_number': 1,
             'episode_id': '3334',
@@ -75,7 +74,6 @@ class VVVVIDIE(InfoExtractor):
     def _extract_common_video_info(self, video_data):
         return {
             'thumbnail': video_data.get('thumbnail'),
-            'episode_number': int_or_none(video_data.get('number')),
             'episode_id': str_or_none(video_data.get('id')),
         }
 
@@ -145,6 +143,17 @@ class VVVVIDIE(InfoExtractor):
 
             return d
 
+        info = {}
+
+        def metadata_from_url(r_url):
+            if not info and r_url:
+                mobj = re.search(r'_(?:S(\d+))?Ep(\d+)', r_url)
+                if mobj:
+                    info['episode_number'] = int(mobj.group(2))
+                    season_number = mobj.group(1)
+                    if season_number:
+                        info['season_number'] = int(season_number)
+
         for quality in ('_sd', ''):
             embed_code = video_data.get('embed_info' + quality)
             if not embed_code:
@@ -166,9 +175,12 @@ class VVVVIDIE(InfoExtractor):
             else:
                 formats.extend(self._extract_wowza_formats(
                     'http://sb.top-ix.org/videomg/_definst_/mp4:%s/playlist.m3u8' % embed_code, video_id))
+            metadata_from_url(embed_code)
+
         self._sort_formats(formats)
 
-        info = self._extract_common_video_info(video_data)
+        metadata_from_url(video_data.get('thumbnail'))
+        info.update(self._extract_common_video_info(video_data))
         info.update({
             'id': video_id,
             'title': title,
@@ -176,7 +188,6 @@ class VVVVIDIE(InfoExtractor):
             'duration': int_or_none(video_data.get('length')),
             'series': video_data.get('show_title'),
             'season_id': season_id,
-            'season_number': video_data.get('season_number'),
             'episode': title,
             'view_count': int_or_none(video_data.get('views')),
             'like_count': int_or_none(video_data.get('video_likes')),
@@ -211,7 +222,6 @@ class VVVVIDShowIE(VVVVIDIE):
 
         entries = []
         for season in (seasons or []):
-            season_number = int_or_none(season.get('number'))
             episodes = season.get('episodes') or []
             for episode in episodes:
                 if episode.get('playable') is False:
@@ -227,7 +237,6 @@ class VVVVIDShowIE(VVVVIDIE):
                     'url': '/'.join([base_url, season_id, video_id]),
                     'title': episode.get('title'),
                     'description': episode.get('description'),
-                    'season_number': season_number,
                     'season_id': season_id,
                 })
                 entries.append(info)

From 2df93a0c4ada8eff399844ac4a249a743e0a980d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jan 2021 05:01:45 +0700
Subject: [PATCH 046/860] [nrktv] Switch to playback endpoint

mediaelement endpoint is no longer in use.
---
 youtube_dl/extractor/nrk.py | 273 ++++++++----------------------------
 1 file changed, 57 insertions(+), 216 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index b545f291b..871e4845c 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -6,15 +6,11 @@ import random
 import re
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_str,
-    compat_urllib_parse_unquote,
-)
+from ..compat import compat_str
 from ..utils import (
     determine_ext,
     ExtractorError,
     int_or_none,
-    parse_age_limit,
     parse_duration,
     try_get,
     urljoin,
@@ -63,60 +59,8 @@ class NRKBaseIE(InfoExtractor):
         return self._download_json(
             urljoin('http://psapi.nrk.no/', path),
             video_id, note or 'Downloading %s JSON' % item,
-            fatal=fatal, query=query)
-
-
-class NRKIE(NRKBaseIE):
-    _VALID_URL = r'''(?x)
-                        (?:
-                            nrk:|
-                            https?://
-                                (?:
-                                    (?:www\.)?nrk\.no/video/(?:PS\*|[^_]+_)|
-                                    v8[-.]psapi\.nrk\.no/mediaelement/
-                                )
-                            )
-                            (?P<id>[^?\#&]+)
-                        '''
-
-    _TESTS = [{
-        # video
-        'url': 'http://www.nrk.no/video/PS*150533',
-        'md5': 'f46be075326e23ad0e524edfcb06aeb6',
-        'info_dict': {
-            'id': '150533',
-            'ext': 'mp4',
-            'title': 'Dompap og andre fugler i Piip-Show',
-            'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f',
-            'duration': 262,
-        }
-    }, {
-        # audio
-        'url': 'http://www.nrk.no/video/PS*154915',
-        # MD5 is unstable
-        'info_dict': {
-            'id': '154915',
-            'ext': 'mp4',
-            'title': 'Slik høres internett ut når du er blind',
-            'description': 'md5:a621f5cc1bd75c8d5104cb048c6b8568',
-            'duration': 20,
-        }
-    }, {
-        'url': 'nrk:ecc1b952-96dc-4a98-81b9-5296dc7a98d9',
-        'only_matching': True,
-    }, {
-        'url': 'nrk:clip/7707d5a3-ebe7-434a-87d5-a3ebe7a34a70',
-        'only_matching': True,
-    }, {
-        'url': 'https://v8-psapi.nrk.no/mediaelement/ecc1b952-96dc-4a98-81b9-5296dc7a98d9',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.nrk.no/video/dompap-og-andre-fugler-i-piip-show_150533',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.nrk.no/video/humor/kommentatorboksen-reiser-til-sjos_d1fda11f-a4ad-437a-a374-0398bc84e999',
-        'only_matching': True,
-    }]
+            fatal=fatal, query=query,
+            headers={'Accept-Encoding': 'gzip, deflate, br'})
 
     def _extract_from_playback(self, video_id):
         path_templ = 'playback/%s/' + video_id
@@ -178,6 +122,59 @@ class NRKIE(NRKBaseIE):
             'formats': formats,
         }
 
+
+class NRKIE(NRKBaseIE):
+    _VALID_URL = r'''(?x)
+                        (?:
+                            nrk:|
+                            https?://
+                                (?:
+                                    (?:www\.)?nrk\.no/video/(?:PS\*|[^_]+_)|
+                                    v8[-.]psapi\.nrk\.no/mediaelement/
+                                )
+                            )
+                            (?P<id>[^?\#&]+)
+                        '''
+
+    _TESTS = [{
+        # video
+        'url': 'http://www.nrk.no/video/PS*150533',
+        'md5': 'f46be075326e23ad0e524edfcb06aeb6',
+        'info_dict': {
+            'id': '150533',
+            'ext': 'mp4',
+            'title': 'Dompap og andre fugler i Piip-Show',
+            'description': 'md5:d9261ba34c43b61c812cb6b0269a5c8f',
+            'duration': 262,
+        }
+    }, {
+        # audio
+        'url': 'http://www.nrk.no/video/PS*154915',
+        # MD5 is unstable
+        'info_dict': {
+            'id': '154915',
+            'ext': 'mp4',
+            'title': 'Slik høres internett ut når du er blind',
+            'description': 'md5:a621f5cc1bd75c8d5104cb048c6b8568',
+            'duration': 20,
+        }
+    }, {
+        'url': 'nrk:ecc1b952-96dc-4a98-81b9-5296dc7a98d9',
+        'only_matching': True,
+    }, {
+        'url': 'nrk:clip/7707d5a3-ebe7-434a-87d5-a3ebe7a34a70',
+        'only_matching': True,
+    }, {
+        'url': 'https://v8-psapi.nrk.no/mediaelement/ecc1b952-96dc-4a98-81b9-5296dc7a98d9',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.nrk.no/video/dompap-og-andre-fugler-i-piip-show_150533',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.nrk.no/video/humor/kommentatorboksen-reiser-til-sjos_d1fda11f-a4ad-437a-a374-0398bc84e999',
+        'only_matching': True,
+    }]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         return self._extract_from_playback(video_id)
@@ -187,7 +184,6 @@ class NRKTVIE(NRKBaseIE):
     IE_DESC = 'NRK TV and NRK Radio'
     _EPISODE_RE = r'(?P<id>[a-zA-Z]{4}\d{8})'
     _VALID_URL = r'https?://(?:tv|radio)\.nrk(?:super)?\.no/(?:[^/]+/)*%s' % _EPISODE_RE
-    _API_HOSTS = ('psapi-ne.nrk.no', 'psapi-we.nrk.no')
     _TESTS = [{
         'url': 'https://tv.nrk.no/program/MDDP12000117',
         'md5': 'c4a5960f1b00b40d47db65c1064e0ab1',
@@ -290,164 +286,9 @@ class NRKTVIE(NRKBaseIE):
         'only_matching': True,
     }]
 
-    _api_host = None
-
-    def _extract_from_mediaelement(self, video_id):
-        api_hosts = (self._api_host, ) if self._api_host else self._API_HOSTS
-
-        for api_host in api_hosts:
-            data = self._download_json(
-                'http://%s/mediaelement/%s' % (api_host, video_id),
-                video_id, 'Downloading mediaelement JSON',
-                fatal=api_host == api_hosts[-1])
-            if not data:
-                continue
-            self._api_host = api_host
-            break
-
-        title = data.get('fullTitle') or data.get('mainTitle') or data['title']
-        video_id = data.get('id') or video_id
-
-        urls = []
-        entries = []
-
-        conviva = data.get('convivaStatistics') or {}
-        live = (data.get('mediaElementType') == 'Live'
-                or data.get('isLive') is True or conviva.get('isLive'))
-
-        def make_title(t):
-            return self._live_title(t) if live else t
-
-        media_assets = data.get('mediaAssets')
-        if media_assets and isinstance(media_assets, list):
-            def video_id_and_title(idx):
-                return ((video_id, title) if len(media_assets) == 1
-                        else ('%s-%d' % (video_id, idx), '%s (Part %d)' % (title, idx)))
-            for num, asset in enumerate(media_assets, 1):
-                asset_url = asset.get('url')
-                if not asset_url or asset_url in urls:
-                    continue
-                urls.append(asset_url)
-                formats = self._extract_nrk_formats(asset_url, video_id)
-                if not formats:
-                    continue
-                self._sort_formats(formats)
-
-                entry_id, entry_title = video_id_and_title(num)
-                duration = parse_duration(asset.get('duration'))
-                subtitles = {}
-                for subtitle in ('webVtt', 'timedText'):
-                    subtitle_url = asset.get('%sSubtitlesUrl' % subtitle)
-                    if subtitle_url:
-                        subtitles.setdefault('no', []).append({
-                            'url': compat_urllib_parse_unquote(subtitle_url)
-                        })
-                entries.append({
-                    'id': asset.get('carrierId') or entry_id,
-                    'title': make_title(entry_title),
-                    'duration': duration,
-                    'subtitles': subtitles,
-                    'formats': formats,
-                    'is_live': live,
-                })
-
-        if not entries:
-            media_url = data.get('mediaUrl')
-            if media_url and media_url not in urls:
-                formats = self._extract_nrk_formats(media_url, video_id)
-                if formats:
-                    self._sort_formats(formats)
-                    duration = parse_duration(data.get('duration'))
-                    entries = [{
-                        'id': video_id,
-                        'title': make_title(title),
-                        'duration': duration,
-                        'formats': formats,
-                        'is_live': live,
-                    }]
-
-        if not entries:
-            self._raise_error(data)
-
-        series = conviva.get('seriesName') or data.get('seriesTitle')
-        episode = conviva.get('episodeName') or data.get('episodeNumberOrDate')
-
-        season_number = None
-        episode_number = None
-        if data.get('mediaElementType') == 'Episode':
-            _season_episode = data.get('scoresStatistics', {}).get('springStreamStream') or \
-                data.get('relativeOriginUrl', '')
-            EPISODENUM_RE = [
-                r'/s(?P<season>\d{,2})e(?P<episode>\d{,2})\.',
-                r'/sesong-(?P<season>\d{,2})/episode-(?P<episode>\d{,2})',
-            ]
-            season_number = int_or_none(self._search_regex(
-                EPISODENUM_RE, _season_episode, 'season number',
-                default=None, group='season'))
-            episode_number = int_or_none(self._search_regex(
-                EPISODENUM_RE, _season_episode, 'episode number',
-                default=None, group='episode'))
-
-        thumbnails = None
-        images = data.get('images')
-        if images and isinstance(images, dict):
-            web_images = images.get('webImages')
-            if isinstance(web_images, list):
-                thumbnails = [{
-                    'url': image['imageUrl'],
-                    'width': int_or_none(image.get('width')),
-                    'height': int_or_none(image.get('height')),
-                } for image in web_images if image.get('imageUrl')]
-
-        description = data.get('description')
-        category = data.get('mediaAnalytics', {}).get('category')
-
-        common_info = {
-            'description': description,
-            'series': series,
-            'episode': episode,
-            'season_number': season_number,
-            'episode_number': episode_number,
-            'categories': [category] if category else None,
-            'age_limit': parse_age_limit(data.get('legalAge')),
-            'thumbnails': thumbnails,
-        }
-
-        vcodec = 'none' if data.get('mediaType') == 'Audio' else None
-
-        for entry in entries:
-            entry.update(common_info)
-            for f in entry['formats']:
-                f['vcodec'] = vcodec
-
-        points = data.get('shortIndexPoints')
-        if isinstance(points, list):
-            chapters = []
-            for next_num, point in enumerate(points, start=1):
-                if not isinstance(point, dict):
-                    continue
-                start_time = parse_duration(point.get('startPoint'))
-                if start_time is None:
-                    continue
-                end_time = parse_duration(
-                    data.get('duration')
-                    if next_num == len(points)
-                    else points[next_num].get('startPoint'))
-                if end_time is None:
-                    continue
-                chapters.append({
-                    'start_time': start_time,
-                    'end_time': end_time,
-                    'title': point.get('title'),
-                })
-            if chapters and len(entries) == 1:
-                entries[0]['chapters'] = chapters
-
-        return self.playlist_result(entries, video_id, title, description)
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        return self._extract_from_mediaelement(video_id)
+        return self._extract_from_playback(video_id)
 
 
 class NRKTVEpisodeIE(InfoExtractor):

From 21a42e2588226b781a02124e26d709a9562c8fef Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jan 2021 06:29:52 +0700
Subject: [PATCH 047/860] [nrk] Improve extraction (closes #27634, closes
 #27635)

+ Add support for mp3 formats
* Generalize and delegate all item extractors to nrk, beware ie key breakages
+ Add support for podcasts
+ Generalize nrk shortcut form to support all kind of ids
---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/nrk.py        | 248 ++++++++++++++++++++---------
 2 files changed, 172 insertions(+), 77 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d9d3f4940..74743a449 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -790,6 +790,7 @@ from .nrk import (
     NRKSkoleIE,
     NRKTVIE,
     NRKTVDirekteIE,
+    NRKRadioPodkastIE,
     NRKTVEpisodeIE,
     NRKTVEpisodesIE,
     NRKTVSeasonIE,
diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 871e4845c..9621522d4 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -62,66 +62,6 @@ class NRKBaseIE(InfoExtractor):
             fatal=fatal, query=query,
             headers={'Accept-Encoding': 'gzip, deflate, br'})
 
-    def _extract_from_playback(self, video_id):
-        path_templ = 'playback/%s/' + video_id
-
-        def call_playback_api(item, query=None):
-            return self._call_api(path_templ % item, video_id, item, query=query)
-        # known values for preferredCdn: akamai, iponly, minicdn and telenor
-        manifest = call_playback_api('manifest', {'preferredCdn': 'akamai'})
-
-        if manifest.get('playability') == 'nonPlayable':
-            self._raise_error(manifest['nonPlayable'])
-
-        playable = manifest['playable']
-
-        formats = []
-        for asset in playable['assets']:
-            if not isinstance(asset, dict):
-                continue
-            if asset.get('encrypted'):
-                continue
-            format_url = url_or_none(asset.get('url'))
-            if not format_url:
-                continue
-            if asset.get('format') == 'HLS' or determine_ext(format_url) == 'm3u8':
-                formats.extend(self._extract_nrk_formats(format_url, video_id))
-        self._sort_formats(formats)
-
-        data = call_playback_api('metadata')
-
-        preplay = data['preplay']
-        titles = preplay['titles']
-        title = titles['title']
-        alt_title = titles.get('subtitle')
-
-        description = preplay.get('description')
-        duration = parse_duration(playable.get('duration')) or parse_duration(data.get('duration'))
-
-        thumbnails = []
-        for image in try_get(
-                preplay, lambda x: x['poster']['images'], list) or []:
-            if not isinstance(image, dict):
-                continue
-            image_url = url_or_none(image.get('url'))
-            if not image_url:
-                continue
-            thumbnails.append({
-                'url': image_url,
-                'width': int_or_none(image.get('pixelWidth')),
-                'height': int_or_none(image.get('pixelHeight')),
-            })
-
-        return {
-            'id': video_id,
-            'title': title,
-            'alt_title': alt_title,
-            'description': description,
-            'duration': duration,
-            'thumbnails': thumbnails,
-            'formats': formats,
-        }
-
 
 class NRKIE(NRKBaseIE):
     _VALID_URL = r'''(?x)
@@ -173,14 +113,97 @@ class NRKIE(NRKBaseIE):
     }, {
         'url': 'https://www.nrk.no/video/humor/kommentatorboksen-reiser-til-sjos_d1fda11f-a4ad-437a-a374-0398bc84e999',
         'only_matching': True,
+    }, {
+        # podcast
+        'url': 'nrk:l_96f4f1b0-de54-4e6a-b4f1-b0de54fe6af8',
+        'only_matching': True,
+    }, {
+        # clip
+        'url': 'nrk:150533',
+        'only_matching': True,
+    }, {
+        # episode
+        'url': 'nrk:MDDP12000117',
+        'only_matching': True,
+    }, {
+        # direkte
+        'url': 'nrk:nrk1',
+        'only_matching': True,
     }]
 
+    def _extract_from_playback(self, video_id):
+        path_templ = 'playback/%s/' + video_id
+
+        def call_playback_api(item, query=None):
+            return self._call_api(path_templ % item, video_id, item, query=query)
+        # known values for preferredCdn: akamai, iponly, minicdn and telenor
+        manifest = call_playback_api('manifest', {'preferredCdn': 'akamai'})
+
+        if manifest.get('playability') == 'nonPlayable':
+            self._raise_error(manifest['nonPlayable'])
+
+        playable = manifest['playable']
+
+        formats = []
+        for asset in playable['assets']:
+            if not isinstance(asset, dict):
+                continue
+            if asset.get('encrypted'):
+                continue
+            format_url = url_or_none(asset.get('url'))
+            if not format_url:
+                continue
+            asset_format = (asset.get('format') or '').lower()
+            if asset_format == 'hls' or determine_ext(format_url) == 'm3u8':
+                formats.extend(self._extract_nrk_formats(format_url, video_id))
+            elif asset_format == 'mp3':
+                formats.append({
+                    'url': format_url,
+                    'format_id': asset_format,
+                    'vcodec': 'none',
+                })
+        self._sort_formats(formats)
+
+        data = call_playback_api('metadata')
+
+        preplay = data['preplay']
+        titles = preplay['titles']
+        title = titles['title']
+        alt_title = titles.get('subtitle')
+
+        description = preplay.get('description')
+        duration = parse_duration(playable.get('duration')) or parse_duration(data.get('duration'))
+
+        thumbnails = []
+        for image in try_get(
+                preplay, lambda x: x['poster']['images'], list) or []:
+            if not isinstance(image, dict):
+                continue
+            image_url = url_or_none(image.get('url'))
+            if not image_url:
+                continue
+            thumbnails.append({
+                'url': image_url,
+                'width': int_or_none(image.get('pixelWidth')),
+                'height': int_or_none(image.get('pixelHeight')),
+            })
+
+        return {
+            'id': video_id,
+            'title': title,
+            'alt_title': alt_title,
+            'description': description,
+            'duration': duration,
+            'thumbnails': thumbnails,
+            'formats': formats,
+        }
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         return self._extract_from_playback(video_id)
 
 
-class NRKTVIE(NRKBaseIE):
+class NRKTVIE(InfoExtractor):
     IE_DESC = 'NRK TV and NRK Radio'
     _EPISODE_RE = r'(?P<id>[a-zA-Z]{4}\d{8})'
     _VALID_URL = r'https?://(?:tv|radio)\.nrk(?:super)?\.no/(?:[^/]+/)*%s' % _EPISODE_RE
@@ -288,7 +311,8 @@ class NRKTVIE(NRKBaseIE):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        return self._extract_from_playback(video_id)
+        return self.url_result(
+            'nrk:%s' % video_id, ie=NRKIE.ie_key(), video_id=video_id)
 
 
 class NRKTVEpisodeIE(InfoExtractor):
@@ -359,8 +383,6 @@ class NRKTVSerieBaseIE(NRKBaseIE):
             nrk_id = episode.get('prfId') or episode.get('episodeId')
             if not nrk_id or not isinstance(nrk_id, compat_str):
                 continue
-            if not re.match(NRKTVIE._EPISODE_RE, nrk_id):
-                continue
             entries.append(self.url_result(
                 'nrk:%s' % nrk_id, ie=NRKIE.ie_key(), video_id=nrk_id))
         return entries
@@ -372,6 +394,10 @@ class NRKTVSerieBaseIE(NRKBaseIE):
             if embedded.get(asset_key):
                 return asset_key
 
+    @staticmethod
+    def _catalog_name(serie_kind):
+        return 'podcast' if serie_kind in ('podcast', 'podkast') else 'series'
+
     def _entries(self, data, display_id):
         for page_num in itertools.count(1):
             embedded = data.get('_embedded') or data
@@ -405,7 +431,16 @@ class NRKTVSerieBaseIE(NRKBaseIE):
 
 
 class NRKTVSeasonIE(NRKTVSerieBaseIE):
-    _VALID_URL = r'https?://(?P<domain>tv|radio)\.nrk\.no/serie/(?P<serie>[^/]+)/(?:sesong/)?(?P<id>\d+)'
+    _VALID_URL = r'''(?x)
+                    https?://
+                        (?P<domain>tv|radio)\.nrk\.no/
+                        (?P<serie_kind>serie|pod[ck]ast)/
+                        (?P<serie>[^/]+)/
+                        (?:
+                            (?:sesong/)?(?P<id>\d+)|
+                            sesong/(?P<id_2>[^/?#&]+)
+                        )
+                    '''
     _TESTS = [{
         'url': 'https://tv.nrk.no/serie/backstage/sesong/1',
         'info_dict': {
@@ -441,19 +476,34 @@ class NRKTVSeasonIE(NRKTVSerieBaseIE):
         # 180 entries, single page
         'url': 'https://tv.nrk.no/serie/spangas/sesong/1',
         'only_matching': True,
+    }, {
+        'url': 'https://radio.nrk.no/podkast/hele_historien/sesong/diagnose-kverulant',
+        'info_dict': {
+            'id': 'hele_historien/diagnose-kverulant',
+            'title': 'Diagnose kverulant',
+        },
+        'playlist_mincount': 3,
+    }, {
+        'url': 'https://radio.nrk.no/podkast/loerdagsraadet/sesong/202101',
+        'only_matching': True,
     }]
 
     @classmethod
     def suitable(cls, url):
-        return (False if NRKTVIE.suitable(url) or NRKTVEpisodeIE.suitable(url)
+        return (False if NRKTVIE.suitable(url) or NRKTVEpisodeIE.suitable(url) or NRKRadioPodkastIE.suitable(url)
                 else super(NRKTVSeasonIE, cls).suitable(url))
 
     def _real_extract(self, url):
-        domain, serie, season_id = re.match(self._VALID_URL, url).groups()
+        mobj = re.match(self._VALID_URL, url)
+        domain = mobj.group('domain')
+        serie_kind = mobj.group('serie_kind')
+        serie = mobj.group('serie')
+        season_id = mobj.group('id') or mobj.group('id_2')
         display_id = '%s/%s' % (serie, season_id)
 
         data = self._call_api(
-            '%s/catalog/series/%s/seasons/%s' % (domain, serie, season_id),
+            '%s/catalog/%s/%s/seasons/%s'
+            % (domain, self._catalog_name(serie_kind), serie, season_id),
             display_id, 'season', query={'pageSize': 50})
 
         title = try_get(data, lambda x: x['titles']['title'], compat_str) or display_id
@@ -463,7 +513,7 @@ class NRKTVSeasonIE(NRKTVSerieBaseIE):
 
 
 class NRKTVSeriesIE(NRKTVSerieBaseIE):
-    _VALID_URL = r'https?://(?P<domain>(?:tv|radio)\.nrk|(?:tv\.)?nrksuper)\.no/serie/(?P<id>[^/]+)'
+    _VALID_URL = r'https?://(?P<domain>(?:tv|radio)\.nrk|(?:tv\.)?nrksuper)\.no/(?P<serie_kind>serie|pod[ck]ast)/(?P<id>[^/]+)'
     _TESTS = [{
         # new layout, instalments
         'url': 'https://tv.nrk.no/serie/groenn-glede',
@@ -523,23 +573,33 @@ class NRKTVSeriesIE(NRKTVSerieBaseIE):
     }, {
         'url': 'https://nrksuper.no/serie/labyrint',
         'only_matching': True,
+    }, {
+        'url': 'https://radio.nrk.no/podkast/ulrikkes_univers',
+        'info_dict': {
+            'id': 'ulrikkes_univers',
+        },
+        'playlist_mincount': 10,
+    }, {
+        'url': 'https://radio.nrk.no/podkast/ulrikkes_univers/nrkno-poddkast-26588-134079-05042018030000',
+        'only_matching': True,
     }]
 
     @classmethod
     def suitable(cls, url):
         return (
             False if any(ie.suitable(url)
-                         for ie in (NRKTVIE, NRKTVEpisodeIE, NRKTVSeasonIE))
+                         for ie in (NRKTVIE, NRKTVEpisodeIE, NRKRadioPodkastIE, NRKTVSeasonIE))
             else super(NRKTVSeriesIE, cls).suitable(url))
 
     def _real_extract(self, url):
-        site, series_id = re.match(self._VALID_URL, url).groups()
+        site, serie_kind, series_id = re.match(self._VALID_URL, url).groups()
         is_radio = site == 'radio.nrk'
         domain = 'radio' if is_radio else 'tv'
 
         size_prefix = 'p' if is_radio else 'embeddedInstalmentsP'
         series = self._call_api(
-            '%s/catalog/series/%s' % (domain, series_id),
+            '%s/catalog/%s/%s'
+            % (domain, self._catalog_name(serie_kind), series_id),
             series_id, 'serie', query={size_prefix + 'ageSize': 50})
         titles = try_get(series, [
             lambda x: x['titles'],
@@ -554,12 +614,14 @@ class NRKTVSeriesIE(NRKTVSerieBaseIE):
         embedded_seasons = embedded.get('seasons') or []
         if len(linked_seasons) > len(embedded_seasons):
             for season in linked_seasons:
-                season_name = season.get('name')
-                if season_name and isinstance(season_name, compat_str):
+                season_url = urljoin(url, season.get('href'))
+                if not season_url:
+                    season_name = season.get('name')
+                    if season_name and isinstance(season_name, compat_str):
+                        season_url = 'https://%s.nrk.no/serie/%s/sesong/%s' % (domain, series_id, season_name)
+                if season_url:
                     entries.append(self.url_result(
-                        'https://%s.nrk.no/serie/%s/sesong/%s'
-                        % (domain, series_id, season_name),
-                        ie=NRKTVSeasonIE.ie_key(),
+                        season_url, ie=NRKTVSeasonIE.ie_key(),
                         video_title=season.get('title')))
         else:
             for season in embedded_seasons:
@@ -584,6 +646,38 @@ class NRKTVDirekteIE(NRKTVIE):
     }]
 
 
+class NRKRadioPodkastIE(InfoExtractor):
+    _VALID_URL = r'https?://radio\.nrk\.no/pod[ck]ast/(?:[^/]+/)+(?P<id>l_[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
+
+    _TESTS = [{
+        'url': 'https://radio.nrk.no/podkast/ulrikkes_univers/l_96f4f1b0-de54-4e6a-b4f1-b0de54fe6af8',
+        'md5': '8d40dab61cea8ab0114e090b029a0565',
+        'info_dict': {
+            'id': 'MUHH48000314AA',
+            'ext': 'mp4',
+            'title': '20 spørsmål 23.05.2014',
+            'description': 'md5:bdea103bc35494c143c6a9acdd84887a',
+            'duration': 1741,
+            'series': '20 spørsmål',
+            'episode': '23.05.2014',
+        },
+    }, {
+        'url': 'https://radio.nrk.no/podcast/ulrikkes_univers/l_96f4f1b0-de54-4e6a-b4f1-b0de54fe6af8',
+        'only_matching': True,
+    }, {
+        'url': 'https://radio.nrk.no/podkast/ulrikkes_univers/sesong/1/l_96f4f1b0-de54-4e6a-b4f1-b0de54fe6af8',
+        'only_matching': True,
+    }, {
+        'url': 'https://radio.nrk.no/podkast/hele_historien/sesong/bortfoert-i-bergen/l_774d1a2c-7aa7-4965-8d1a-2c7aa7d9652c',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        return self.url_result(
+            'nrk:%s' % video_id, ie=NRKIE.ie_key(), video_id=video_id)
+
+
 class NRKPlaylistBaseIE(InfoExtractor):
     def _extract_description(self, webpage):
         pass

From 1f1d01d498fee8120bb3415b1b1a20e21259bbf4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jan 2021 06:41:37 +0700
Subject: [PATCH 048/860] [nrk] Add more shortcut tests

---
 youtube_dl/extractor/nrk.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 9621522d4..61a7c9aad 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -117,18 +117,30 @@ class NRKIE(NRKBaseIE):
         # podcast
         'url': 'nrk:l_96f4f1b0-de54-4e6a-b4f1-b0de54fe6af8',
         'only_matching': True,
+    }, {
+        'url': 'nrk:podcast/l_96f4f1b0-de54-4e6a-b4f1-b0de54fe6af8',
+        'only_matching': True,
     }, {
         # clip
         'url': 'nrk:150533',
         'only_matching': True,
     }, {
-        # episode
+        'url': 'nrk:clip/150533',
+        'only_matching': True,
+    }, {
+        # program
         'url': 'nrk:MDDP12000117',
         'only_matching': True,
+    }, {
+        'url': 'nrk:program/ENRK10100318',
+        'only_matching': True,
     }, {
         # direkte
         'url': 'nrk:nrk1',
         'only_matching': True,
+    }, {
+        'url': 'nrk:channel/nrk1',
+        'only_matching': True,
     }]
 
     def _extract_from_playback(self, video_id):

From 7b643d4cd0f5aa02149e29d8212acd42038da63b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jan 2021 06:44:28 +0700
Subject: [PATCH 049/860] [nrk] Improve video id extraction

---
 youtube_dl/extractor/nrk.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 61a7c9aad..5f12b0d9e 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -151,6 +151,8 @@ class NRKIE(NRKBaseIE):
         # known values for preferredCdn: akamai, iponly, minicdn and telenor
         manifest = call_playback_api('manifest', {'preferredCdn': 'akamai'})
 
+        video_id = try_get(manifest, lambda x: x['id'], compat_str) or video_id
+
         if manifest.get('playability') == 'nonPlayable':
             self._raise_error(manifest['nonPlayable'])
 
@@ -211,7 +213,7 @@ class NRKIE(NRKBaseIE):
         }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        video_id = self._match_id(url).split('/')[-1]
         return self._extract_from_playback(video_id)
 
 

From cabfd4b1f0354518068bbdf0718cd36497d5c8a3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jan 2021 06:45:17 +0700
Subject: [PATCH 050/860] [nrk] Inline _extract_from_playback

---
 youtube_dl/extractor/nrk.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 5f12b0d9e..520206534 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -143,7 +143,9 @@ class NRKIE(NRKBaseIE):
         'only_matching': True,
     }]
 
-    def _extract_from_playback(self, video_id):
+    def _real_extract(self, url):
+        video_id = self._match_id(url).split('/')[-1]
+
         path_templ = 'playback/%s/' + video_id
 
         def call_playback_api(item, query=None):
@@ -212,10 +214,6 @@ class NRKIE(NRKBaseIE):
             'formats': formats,
         }
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url).split('/')[-1]
-        return self._extract_from_playback(video_id)
-
 
 class NRKTVIE(InfoExtractor):
     IE_DESC = 'NRK TV and NRK Radio'

From a25d03d7cb52111fab906c2c180f9bdf2c52ed12 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jan 2021 06:51:21 +0700
Subject: [PATCH 051/860] [nrk] Fix age limit extraction

---
 youtube_dl/extractor/nrk.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 520206534..d023de7f7 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -204,6 +204,9 @@ class NRKIE(NRKBaseIE):
                 'height': int_or_none(image.get('pixelHeight')),
             })
 
+        age_limit = int_or_none(try_get(
+            data, lambda x: x['legalAge']['body']['rating']['code']))
+
         return {
             'id': video_id,
             'title': title,
@@ -211,6 +214,7 @@ class NRKIE(NRKBaseIE):
             'description': description,
             'duration': duration,
             'thumbnails': thumbnails,
+            'age_limit': age_limit,
             'formats': formats,
         }
 

From 7dfd966848e17e3d438e5e49e1088b8b1444f9e5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jan 2021 07:02:13 +0700
Subject: [PATCH 052/860] [nrk] Extract subtitles

---
 youtube_dl/extractor/nrk.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index d023de7f7..bd96d9d14 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -12,6 +12,7 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     parse_duration,
+    str_or_none,
     try_get,
     urljoin,
     url_or_none,
@@ -204,6 +205,21 @@ class NRKIE(NRKBaseIE):
                 'height': int_or_none(image.get('pixelHeight')),
             })
 
+        subtitles = {}
+        for sub in try_get(playable, lambda x: x['subtitles'], list) or []:
+            if not isinstance(sub, dict):
+                continue
+            sub_url = url_or_none(sub.get('webVtt'))
+            if not sub_url:
+                continue
+            sub_key = str_or_none(sub.get('language')) or 'nb'
+            sub_type = str_or_none(sub.get('type'))
+            if sub_type:
+                sub_key += '-%s' % sub_type
+            subtitles.setdefault(sub_key, []).append({
+                'url': sub_url,
+            })
+
         age_limit = int_or_none(try_get(
             data, lambda x: x['legalAge']['body']['rating']['code']))
 
@@ -216,6 +232,7 @@ class NRKIE(NRKBaseIE):
             'thumbnails': thumbnails,
             'age_limit': age_limit,
             'formats': formats,
+            'subtitles': subtitles,
         }
 
 

From 85de33b04ef339005cea949576dda4b553296f50 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jan 2021 07:30:45 +0700
Subject: [PATCH 053/860] [nrk] Improve series metadata extraction

---
 youtube_dl/extractor/nrk.py | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index bd96d9d14..20a5d7673 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -223,7 +223,9 @@ class NRKIE(NRKBaseIE):
         age_limit = int_or_none(try_get(
             data, lambda x: x['legalAge']['body']['rating']['code']))
 
-        return {
+        is_series = try_get(data, lambda x: x['_links']['series']['name']) == 'series'
+
+        info = {
             'id': video_id,
             'title': title,
             'alt_title': alt_title,
@@ -235,6 +237,27 @@ class NRKIE(NRKBaseIE):
             'subtitles': subtitles,
         }
 
+        if is_series:
+            series = title
+            if alt_title:
+                title += ' - %s' % alt_title
+            season_number = int_or_none(self._search_regex(
+                r'Sesong\s+(\d+)', description or '', 'season number',
+                default=None))
+            episode = alt_title if is_series else None
+            episode_number = int_or_none(self._search_regex(
+                r'(\d+)\.\s+episode', episode or '', 'episode number',
+                default=None))
+            info.update({
+                'title': title,
+                'series': series,
+                'season_number': season_number,
+                'episode': episode,
+                'episode_number': episode_number,
+            })
+
+        return info
+
 
 class NRKTVIE(InfoExtractor):
     IE_DESC = 'NRK TV and NRK Radio'

From bc2f83b95e02c9a75e576109ca0bced8a6c67eb2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jan 2021 07:31:03 +0700
Subject: [PATCH 054/860] [nrktv] Fix tests

---
 youtube_dl/extractor/nrk.py | 28 +++++++++++++++-------------
 1 file changed, 15 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 20a5d7673..4fb7df959 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -267,7 +267,7 @@ class NRKTVIE(InfoExtractor):
         'url': 'https://tv.nrk.no/program/MDDP12000117',
         'md5': 'c4a5960f1b00b40d47db65c1064e0ab1',
         'info_dict': {
-            'id': 'MDDP12000117AA',
+            'id': 'MDDP12000117',
             'ext': 'mp4',
             'title': 'Alarm Trolltunga',
             'description': 'md5:46923a6e6510eefcce23d5ef2a58f2ce',
@@ -278,24 +278,25 @@ class NRKTVIE(InfoExtractor):
         'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',
         'md5': '8d40dab61cea8ab0114e090b029a0565',
         'info_dict': {
-            'id': 'MUHH48000314AA',
+            'id': 'MUHH48000314',
             'ext': 'mp4',
-            'title': '20 spørsmål 23.05.2014',
+            'title': '20 spørsmål - 23. mai 2014',
+            'alt_title': '23. mai 2014',
             'description': 'md5:bdea103bc35494c143c6a9acdd84887a',
             'duration': 1741,
             'series': '20 spørsmål',
-            'episode': '23.05.2014',
+            'episode': '23. mai 2014',
         },
     }, {
         'url': 'https://tv.nrk.no/program/mdfp15000514',
         'info_dict': {
-            'id': 'MDFP15000514CA',
+            'id': 'MDFP15000514',
             'ext': 'mp4',
-            'title': 'Grunnlovsjubiléet - Stor ståhei for ingenting 24.05.2014',
+            'title': 'Kunnskapskanalen - Grunnlovsjubiléet - Stor ståhei for ingenting',
             'description': 'md5:89290c5ccde1b3a24bb8050ab67fe1db',
             'duration': 4605.08,
             'series': 'Kunnskapskanalen',
-            'episode': '24.05.2014',
+            'episode': 'Grunnlovsjubiléet - Stor ståhei for ingenting',
         },
         'params': {
             'skip_download': True,
@@ -304,7 +305,7 @@ class NRKTVIE(InfoExtractor):
         # single playlist video
         'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015#del=2',
         'info_dict': {
-            'id': 'MSPO40010515AH',
+            'id': 'MSPO40010515',
             'ext': 'mp4',
             'title': 'Sprint fri teknikk, kvinner og menn 06.01.2015',
             'description': 'md5:c03aba1e917561eface5214020551b7a',
@@ -317,22 +318,23 @@ class NRKTVIE(InfoExtractor):
     }, {
         'url': 'https://tv.nrk.no/serie/tour-de-ski/MSPO40010515/06-01-2015',
         'info_dict': {
-            'id': 'MSPO40010515AH',
+            'id': 'MSPO40010515',
             'ext': 'mp4',
             'title': 'Sprint fri teknikk, kvinner og menn 06.01.2015',
             'description': 'md5:c03aba1e917561eface5214020551b7a',
         },
         'expected_warnings': ['Failed to download m3u8 information'],
+        'skip': 'Ikke tilgjengelig utenfor Norge',
     }, {
         'url': 'https://tv.nrk.no/serie/anno/KMTE50001317/sesong-3/episode-13',
         'info_dict': {
-            'id': 'KMTE50001317AA',
+            'id': 'KMTE50001317',
             'ext': 'mp4',
-            'title': 'Anno 13:30',
+            'title': 'Anno - 13. episode',
             'description': 'md5:11d9613661a8dbe6f9bef54e3a4cbbfa',
             'duration': 2340,
             'series': 'Anno',
-            'episode': '13:30',
+            'episode': '13. episode',
             'season_number': 3,
             'episode_number': 13,
         },
@@ -342,7 +344,7 @@ class NRKTVIE(InfoExtractor):
     }, {
         'url': 'https://tv.nrk.no/serie/nytt-paa-nytt/MUHH46000317/27-01-2017',
         'info_dict': {
-            'id': 'MUHH46000317AA',
+            'id': 'MUHH46000317',
             'ext': 'mp4',
             'title': 'Nytt på Nytt 27.01.2017',
             'description': 'md5:5358d6388fba0ea6f0b6d11c48b9eb4b',

From b2d9fd9c9f7d4b9f6b6bebbbb8f7e799cb08d4b3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jan 2021 07:53:05 +0700
Subject: [PATCH 055/860] [nrk] Improve episode and season number extraction

---
 youtube_dl/extractor/nrk.py | 30 +++++++++++++++++-------------
 1 file changed, 17 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 4fb7df959..48387420c 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -246,7 +246,9 @@ class NRKIE(NRKBaseIE):
                 default=None))
             episode = alt_title if is_series else None
             episode_number = int_or_none(self._search_regex(
-                r'(\d+)\.\s+episode', episode or '', 'episode number',
+                r'^(\d+)\.', episode or '', 'episode number',
+                default=None)) or int_or_none(self._search_regex(
+                r'\((\d+)\s*:\s*\d+\)', description or '', 'episode number',
                 default=None))
             info.update({
                 'title': title,
@@ -374,19 +376,19 @@ class NRKTVIE(InfoExtractor):
 
 
 class NRKTVEpisodeIE(InfoExtractor):
-    _VALID_URL = r'https?://tv\.nrk\.no/serie/(?P<id>[^/]+/sesong/\d+/episode/\d+)'
+    _VALID_URL = r'https?://tv\.nrk\.no/serie/(?P<id>[^/]+/sesong/(?P<season_number>\d+)/episode/(?P<episode_number>\d+))'
     _TESTS = [{
         'url': 'https://tv.nrk.no/serie/hellums-kro/sesong/1/episode/2',
         'info_dict': {
-            'id': 'MUHH36005220BA',
+            'id': 'MUHH36005220',
             'ext': 'mp4',
-            'title': 'Kro, krig og kjærlighet 2:6',
-            'description': 'md5:b32a7dc0b1ed27c8064f58b97bda4350',
-            'duration': 1563,
+            'title': 'Hellums kro - 2. Kro, krig og kjærlighet',
+            'description': 'md5:ad92ddffc04cea8ce14b415deef81787',
+            'duration': 1563.92,
             'series': 'Hellums kro',
-            'season_number': 1,
+            # 'season_number': 1,
             'episode_number': 2,
-            'episode': '2:6',
+            'episode': '2. Kro, krig og kjærlighet',
             'age_limit': 6,
         },
         'params': {
@@ -395,15 +397,15 @@ class NRKTVEpisodeIE(InfoExtractor):
     }, {
         'url': 'https://tv.nrk.no/serie/backstage/sesong/1/episode/8',
         'info_dict': {
-            'id': 'MSUI14000816AA',
+            'id': 'MSUI14000816',
             'ext': 'mp4',
-            'title': 'Backstage 8:30',
+            'title': 'Backstage - 8. episode',
             'description': 'md5:de6ca5d5a2d56849e4021f2bf2850df4',
             'duration': 1320,
             'series': 'Backstage',
             'season_number': 1,
             'episode_number': 8,
-            'episode': '8:30',
+            'episode': '8. episode',
         },
         'params': {
             'skip_download': True,
@@ -412,7 +414,7 @@ class NRKTVEpisodeIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        display_id, season_number, episode_number = re.match(self._VALID_URL, url).groups()
 
         webpage = self._download_webpage(url, display_id)
 
@@ -424,10 +426,12 @@ class NRKTVEpisodeIE(InfoExtractor):
         assert re.match(NRKTVIE._EPISODE_RE, nrk_id)
 
         info.update({
-            '_type': 'url_transparent',
+            '_type': 'url',
             'id': nrk_id,
             'url': 'nrk:%s' % nrk_id,
             'ie_key': NRKIE.ie_key(),
+            'season_number': int(season_number),
+            'episode_number': int(episode_number),
         })
         return info
 

From 1aaee908b9ba12fc363c65daa2ef7d1690472a13 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jan 2021 07:57:12 +0700
Subject: [PATCH 056/860] [nrk] PEP 8

---
 youtube_dl/extractor/nrk.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 48387420c..2873d7938 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -247,9 +247,11 @@ class NRKIE(NRKBaseIE):
             episode = alt_title if is_series else None
             episode_number = int_or_none(self._search_regex(
                 r'^(\d+)\.', episode or '', 'episode number',
-                default=None)) or int_or_none(self._search_regex(
-                r'\((\d+)\s*:\s*\d+\)', description or '', 'episode number',
                 default=None))
+            if not episode_number:
+                episode_number = int_or_none(self._search_regex(
+                    r'\((\d+)\s*:\s*\d+\)', description or '',
+                    'episode number', default=None))
             info.update({
                 'title': title,
                 'series': series,

From 5c6e84c0ff732559d8307e9e823b5a07b0bcc9ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jan 2021 08:31:26 +0700
Subject: [PATCH 057/860] [nrk] Improve series metadata extraction (closes
 #27473)

---
 youtube_dl/extractor/nrk.py | 32 +++++++++++++++++++++++---------
 1 file changed, 23 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 2873d7938..5d33355e7 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -238,16 +238,29 @@ class NRKIE(NRKBaseIE):
         }
 
         if is_series:
-            series = title
+            series = season_id = season_number = episode = episode_number = None
+            programs = self._call_api(
+                'programs/%s' % video_id, video_id, 'programs', fatal=False)
+            if programs and isinstance(programs, dict):
+                series = str_or_none(programs.get('seriesTitle'))
+                season_id = str_or_none(programs.get('seasonId'))
+                season_number = int_or_none(programs.get('seasonNumber'))
+                episode = str_or_none(programs.get('episodeTitle'))
+                episode_number = int_or_none(programs.get('episodeNumber'))
+            if not series:
+                series = title
             if alt_title:
                 title += ' - %s' % alt_title
-            season_number = int_or_none(self._search_regex(
-                r'Sesong\s+(\d+)', description or '', 'season number',
-                default=None))
-            episode = alt_title if is_series else None
-            episode_number = int_or_none(self._search_regex(
-                r'^(\d+)\.', episode or '', 'episode number',
-                default=None))
+            if not season_number:
+                season_number = int_or_none(self._search_regex(
+                    r'Sesong\s+(\d+)', description or '', 'season number',
+                    default=None))
+            if not episode:
+                episode = alt_title if is_series else None
+            if not episode_number:
+                episode_number = int_or_none(self._search_regex(
+                    r'^(\d+)\.', episode or '', 'episode number',
+                    default=None))
             if not episode_number:
                 episode_number = int_or_none(self._search_regex(
                     r'\((\d+)\s*:\s*\d+\)', description or '',
@@ -255,6 +268,7 @@ class NRKIE(NRKBaseIE):
             info.update({
                 'title': title,
                 'series': series,
+                'season_id': season_id,
                 'season_number': season_number,
                 'episode': episode,
                 'episode_number': episode_number,
@@ -388,7 +402,7 @@ class NRKTVEpisodeIE(InfoExtractor):
             'description': 'md5:ad92ddffc04cea8ce14b415deef81787',
             'duration': 1563.92,
             'series': 'Hellums kro',
-            # 'season_number': 1,
+            'season_number': 1,
             'episode_number': 2,
             'episode': '2. Kro, krig og kjærlighet',
             'age_limit': 6,

From d5b8cf093cdcf3ba873dba9aad898a3af91429e1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jan 2021 08:58:05 +0700
Subject: [PATCH 058/860] [nrk] Fix age limit extraction

---
 youtube_dl/extractor/nrk.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 5d33355e7..69178e157 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -220,8 +220,15 @@ class NRKIE(NRKBaseIE):
                 'url': sub_url,
             })
 
-        age_limit = int_or_none(try_get(
-            data, lambda x: x['legalAge']['body']['rating']['code']))
+        legal_age = try_get(
+            data, lambda x: x['legalAge']['body']['rating']['code'], compat_str)
+        # https://en.wikipedia.org/wiki/Norwegian_Media_Authority
+        if legal_age == 'A':
+            age_limit = 0
+        elif legal_age.isdigit():
+            age_limit = int_or_none(legal_age)
+        else:
+            age_limit = None
 
         is_series = try_get(data, lambda x: x['_links']['series']['name']) == 'series'
 
@@ -304,6 +311,7 @@ class NRKTVIE(InfoExtractor):
             'duration': 1741,
             'series': '20 spørsmål',
             'episode': '23. mai 2014',
+            'age_limit': 0,
         },
     }, {
         'url': 'https://tv.nrk.no/program/mdfp15000514',
@@ -315,6 +323,7 @@ class NRKTVIE(InfoExtractor):
             'duration': 4605.08,
             'series': 'Kunnskapskanalen',
             'episode': 'Grunnlovsjubiléet - Stor ståhei for ingenting',
+            'age_limit': 0,
         },
         'params': {
             'skip_download': True,
@@ -327,6 +336,7 @@ class NRKTVIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Sprint fri teknikk, kvinner og menn 06.01.2015',
             'description': 'md5:c03aba1e917561eface5214020551b7a',
+            'age_limit': 0,
         },
         'params': {
             'skip_download': True,
@@ -340,6 +350,7 @@ class NRKTVIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Sprint fri teknikk, kvinner og menn 06.01.2015',
             'description': 'md5:c03aba1e917561eface5214020551b7a',
+            'age_limit': 0,
         },
         'expected_warnings': ['Failed to download m3u8 information'],
         'skip': 'Ikke tilgjengelig utenfor Norge',
@@ -355,6 +366,7 @@ class NRKTVIE(InfoExtractor):
             'episode': '13. episode',
             'season_number': 3,
             'episode_number': 13,
+            'age_limit': 0,
         },
         'params': {
             'skip_download': True,
@@ -369,6 +381,7 @@ class NRKTVIE(InfoExtractor):
             'duration': 1796,
             'series': 'Nytt på nytt',
             'episode': '27.01.2017',
+            'age_limit': 0,
         },
         'params': {
             'skip_download': True,
@@ -422,6 +435,7 @@ class NRKTVEpisodeIE(InfoExtractor):
             'season_number': 1,
             'episode_number': 8,
             'episode': '8. episode',
+            'age_limit': 0,
         },
         'params': {
             'skip_download': True,

From f4afb9a6a8832c48888956d1e8722cb5bd36a78e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jan 2021 09:10:40 +0700
Subject: [PATCH 059/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 3782ad090..4f5fbc7f8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,24 @@
+version <unreleased>
+
+Extractors
+* [nrk] Improve series metadata extraction (#27473)
++ [nrk] Extract subtitles
+* [nrk] Fix age limit extraction
+* [nrk] Improve video id extraction
++ [nrk] Add support for podcasts (#27634, #27635)
+* [nrk] Generalize and delegate all item extractors to nrk
++ [nrk] Add support for mp3 formats
+* [nrktv] Switch to playback endpoint
+* [vvvvid] Fix season metadata extraction (#18130)
+* [stitcher] Fix extraction (#20811, #27606)
+* [acast] Fix extraction (#21444, #27612, #27613)
++ [arcpublishing] Add support for arcpublishing.com (#2298, #9340, #17200)
++ [sky] Add support for Sports News articles and Brighcove videos (#13054)
++ [vvvvid] Extract akamai formats
+* [vvvvid] Skip unplayable episodes (#27599)
+* [yandexvideo] Fix extraction for Python 3.4
+
+
 version 2020.12.31
 
 Core

From 8e953dcbb10a1a42f4e12e4e132657cb0100a1f8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jan 2021 09:12:47 +0700
Subject: [PATCH 060/860] release 2021.01.03

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 docs/supportedsites.md                           | 7 +++++--
 youtube_dl/version.py                            | 2 +-
 8 files changed, 19 insertions(+), 16 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index d1a6ad1f6..ee2da644b 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2020.12.31**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.03**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2020.12.31
+ [debug] youtube-dl version 2021.01.03
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index ded5beadf..7d535b6d1 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2020.12.31**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.03**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 0e06de8dc..74674ce62 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2020.12.31**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.03**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index dfbfe3701..7b0402d41 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2020.12.31**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.03**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2020.12.31
+ [debug] youtube-dl version 2021.01.03
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index 30061808d..17ffff465 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2020.12.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2020.12.31**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.03**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index 4f5fbc7f8..9942ee578 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.01.03
 
 Extractors
 * [nrk] Improve series metadata extraction (#27473)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index b1112f83b..8178af5b6 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -57,6 +57,7 @@
  - **appletrailers**
  - **appletrailers:section**
  - **archive.org**: archive.org videos
+ - **ArcPublishing**
  - **ARD**
  - **ARD:mediathek**
  - **ARDBetaMediathek**
@@ -610,6 +611,7 @@
  - **Npr**
  - **NRK**
  - **NRKPlaylist**
+ - **NRKRadioPodkast**
  - **NRKSkole**: NRK Skole
  - **NRKTV**: NRK TV and NRK Radio
  - **NRKTVDirekte**: NRK TV Direkte and NRK Radio Direkte
@@ -813,12 +815,13 @@
  - **ShowRoomLive**
  - **Sina**
  - **sky.it**
+ - **sky:news**
+ - **sky:sports**
+ - **sky:sports:news**
  - **skyacademy.it**
  - **SkylineWebcams**
- - **SkyNews**
  - **skynewsarabia:article**
  - **skynewsarabia:video**
- - **SkySports**
  - **Slideshare**
  - **SlidesLive**
  - **Slutload**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f795f0735..1588ee8e4 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2020.12.31'
+__version__ = '2021.01.03'

From ac71fd5919302f0d42c0cd79e04522cab8ab0318 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 3 Jan 2021 10:04:32 +0100
Subject: [PATCH 061/860] [stv] improve episode id extraction(closes #23083)

---
 youtube_dl/extractor/stv.py | 42 ++++++++++++++++++++++++++++++-------
 1 file changed, 35 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/stv.py b/youtube_dl/extractor/stv.py
index bae8b71f4..539220a94 100644
--- a/youtube_dl/extractor/stv.py
+++ b/youtube_dl/extractor/stv.py
@@ -8,13 +8,17 @@ from ..utils import (
     compat_str,
     float_or_none,
     int_or_none,
+    smuggle_url,
+    str_or_none,
+    try_get,
 )
 
 
 class STVPlayerIE(InfoExtractor):
     IE_NAME = 'stv:player'
     _VALID_URL = r'https?://player\.stv\.tv/(?P<type>episode|video)/(?P<id>[a-z0-9]{4})'
-    _TEST = {
+    _TESTS = [{
+        # shortform
         'url': 'https://player.stv.tv/video/4gwd/emmerdale/60-seconds-on-set-with-laura-norton/',
         'md5': '5adf9439c31d554f8be0707c7abe7e0a',
         'info_dict': {
@@ -27,7 +31,11 @@ class STVPlayerIE(InfoExtractor):
             'uploader_id': '1486976045',
         },
         'skip': 'this resource is unavailable outside of the UK',
-    }
+    }, {
+        # episodes
+        'url': 'https://player.stv.tv/episode/4125/jennifer-saunders-memory-lane',
+        'only_matching': True,
+    }]
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1486976045/default_default/index.html?videoId=%s'
     _PTYPE_MAP = {
         'episode': 'episodes',
@@ -36,11 +44,31 @@ class STVPlayerIE(InfoExtractor):
 
     def _real_extract(self, url):
         ptype, video_id = re.match(self._VALID_URL, url).groups()
-        resp = self._download_json(
-            'https://player.api.stv.tv/v1/%s/%s' % (self._PTYPE_MAP[ptype], video_id),
-            video_id)
 
-        result = resp['results']
+        webpage = self._download_webpage(url, video_id, fatal=False) or ''
+        props = (self._parse_json(self._search_regex(
+            r'<script[^>]+id="__NEXT_DATA__"[^>]*>({.+?})</script>',
+            webpage, 'next data', default='{}'), video_id,
+            fatal=False) or {}).get('props') or {}
+        player_api_cache = try_get(
+            props, lambda x: x['initialReduxState']['playerApiCache']) or {}
+
+        api_path, resp = None, {}
+        for k, v in player_api_cache.items():
+            if k.startswith('/episodes/') or k.startswith('/shortform/'):
+                api_path, resp = k, v
+                break
+        else:
+            episode_id = str_or_none(try_get(
+                props, lambda x: x['pageProps']['episodeId']))
+            api_path = '/%s/%s' % (self._PTYPE_MAP[ptype], episode_id or video_id)
+
+        result = resp.get('results')
+        if not result:
+            resp = self._download_json(
+                'https://player.api.stv.tv/v1' + api_path, video_id)
+            result = resp['results']
+
         video = result['video']
         video_id = compat_str(video['id'])
 
@@ -57,7 +85,7 @@ class STVPlayerIE(InfoExtractor):
         return {
             '_type': 'url_transparent',
             'id': video_id,
-            'url': self.BRIGHTCOVE_URL_TEMPLATE % video_id,
+            'url': smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % video_id, {'geo_countries': ['GB']}),
             'description': result.get('summary'),
             'duration': float_or_none(video.get('length'), 1000),
             'subtitles': subtitles,

From 8522bcd97c4173407261a3fa0283dd7800c39601 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 3 Jan 2021 12:12:06 +0100
Subject: [PATCH 062/860] [stitcher] Add support for shows and show metadata
 extraction(closes #20510)

---
 youtube_dl/extractor/extractors.py |   5 +-
 youtube_dl/extractor/stitcher.py   | 120 +++++++++++++++++++++--------
 2 files changed, 92 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 74743a449..d1e1e9a60 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1092,7 +1092,10 @@ from .spike import (
     BellatorIE,
     ParamountNetworkIE,
 )
-from .stitcher import StitcherIE
+from .stitcher import (
+    StitcherIE,
+    StitcherShowIE,
+)
 from .sport5 import Sport5IE
 from .sportbox import SportBoxIE
 from .sportdeutschland import SportDeutschlandIE
diff --git a/youtube_dl/extractor/stitcher.py b/youtube_dl/extractor/stitcher.py
index b8b5711b1..3dd0d3b5f 100644
--- a/youtube_dl/extractor/stitcher.py
+++ b/youtube_dl/extractor/stitcher.py
@@ -1,19 +1,60 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     clean_html,
     ExtractorError,
     int_or_none,
     str_or_none,
     try_get,
+    url_or_none,
 )
 
 
-class StitcherIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?stitcher\.com/(?:podcast|show)/(?:[^/]+/)+e(?:pisode)?/(?:(?P<display_id>[^/#?&]+?)-)?(?P<id>\d+)(?:[/#?&]|$)'
+class StitcherBaseIE(InfoExtractor):
+    _VALID_URL_BASE = r'https?://(?:www\.)?stitcher\.com/(?:podcast|show)/'
+
+    def _call_api(self, path, video_id, query):
+        resp = self._download_json(
+            'https://api.prod.stitcher.com/' + path,
+            video_id, query=query)
+        error_massage = try_get(resp, lambda x: x['errors'][0]['message'])
+        if error_massage:
+            raise ExtractorError(error_massage, expected=True)
+        return resp['data']
+
+    def _extract_description(self, data):
+        return clean_html(data.get('html_description') or data.get('description'))
+
+    def _extract_audio_url(self, episode):
+        return url_or_none(episode.get('audio_url') or episode.get('guid'))
+
+    def _extract_show_info(self, show):
+        return {
+            'thumbnail': show.get('image_base_url'),
+            'series': show.get('title'),
+        }
+
+    def _extract_episode(self, episode, audio_url, show_info):
+        info = {
+            'id': compat_str(episode['id']),
+            'display_id': episode.get('slug'),
+            'title': episode['title'].strip(),
+            'description': self._extract_description(episode),
+            'duration': int_or_none(episode.get('duration')),
+            'url': audio_url,
+            'vcodec': 'none',
+            'timestamp': int_or_none(episode.get('date_published')),
+            'season_number': int_or_none(episode.get('season')),
+            'season_id': str_or_none(episode.get('season_id')),
+        }
+        info.update(show_info)
+        return info
+
+
+class StitcherIE(StitcherBaseIE):
+    _VALID_URL = StitcherBaseIE._VALID_URL_BASE + r'(?:[^/]+/)+e(?:pisode)?/(?:[^/#?&]+-)?(?P<id>\d+)'
     _TESTS = [{
         'url': 'http://www.stitcher.com/podcast/the-talking-machines/e/40789481?autoplay=true',
         'md5': 'e9635098e0da10b21a0e2b85585530f6',
@@ -24,8 +65,9 @@ class StitcherIE(InfoExtractor):
             'description': 'md5:547adb4081864be114ae3831b4c2b42f',
             'duration': 1604,
             'thumbnail': r're:^https?://.*\.jpg',
-            'upload_date': '20180126',
-            'timestamp': 1516989316,
+            'upload_date': '20151008',
+            'timestamp': 1444285800,
+            'series': 'Talking Machines',
         },
     }, {
         'url': 'http://www.stitcher.com/podcast/panoply/vulture-tv/e/the-rare-hourlong-comedy-plus-40846275?autoplay=true',
@@ -55,33 +97,47 @@ class StitcherIE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        display_id, audio_id = re.match(self._VALID_URL, url).groups()
+        audio_id = self._match_id(url)
+        data = self._call_api(
+            'shows/episodes', audio_id, {'episode_ids': audio_id})
+        episode = data['episodes'][0]
+        audio_url = self._extract_audio_url(episode)
+        if not audio_url:
+            self.raise_login_required()
+        show = try_get(data, lambda x: x['shows'][0], dict) or {}
+        return self._extract_episode(
+            episode, audio_url, self._extract_show_info(show))
 
-        resp = self._download_json(
-            'https://api.prod.stitcher.com/episode/' + audio_id,
-            display_id or audio_id)
-        episode = try_get(resp, lambda x: x['data']['episodes'][0], dict)
-        if not episode:
-            raise ExtractorError(resp['errors'][0]['message'], expected=True)
 
-        title = episode['title'].strip()
-        audio_url = episode['audio_url']
+class StitcherShowIE(StitcherBaseIE):
+    _VALID_URL = StitcherBaseIE._VALID_URL_BASE + r'(?P<id>[^/#?&]+)/?(?:[?#&]|$)'
+    _TESTS = [{
+        'url': 'http://www.stitcher.com/podcast/the-talking-machines',
+        'info_dict': {
+            'id': 'the-talking-machines',
+            'title': 'Talking Machines',
+            'description': 'md5:831f0995e40f26c10231af39cf1ebf0b',
+        },
+        'playlist_mincount': 106,
+    }, {
+        'url': 'https://www.stitcher.com/show/the-talking-machines',
+        'only_matching': True,
+    }]
 
-        thumbnail = None
-        show_id = episode.get('show_id')
-        if show_id and episode.get('classic_id') != -1:
-            thumbnail = 'https://stitcher-classic.imgix.net/feedimages/%s.jpg' % show_id
+    def _real_extract(self, url):
+        show_slug = self._match_id(url)
+        data = self._call_api(
+            'search/show/%s/allEpisodes' % show_slug, show_slug, {'count': 10000})
+        show = try_get(data, lambda x: x['shows'][0], dict) or {}
+        show_info = self._extract_show_info(show)
 
-        return {
-            'id': audio_id,
-            'display_id': display_id,
-            'title': title,
-            'description': clean_html(episode.get('html_description') or episode.get('description')),
-            'duration': int_or_none(episode.get('duration')),
-            'thumbnail': thumbnail,
-            'url': audio_url,
-            'vcodec': 'none',
-            'timestamp': int_or_none(episode.get('date_created')),
-            'season_number': int_or_none(episode.get('season')),
-            'season_id': str_or_none(episode.get('season_id')),
-        }
+        entries = []
+        for episode in (data.get('episodes') or []):
+            audio_url = self._extract_audio_url(episode)
+            if not audio_url:
+                continue
+            entries.append(self._extract_episode(episode, audio_url, show_info))
+
+        return self.playlist_result(
+            entries, show_slug, show.get('title'),
+            self._extract_description(show))

From 491ee7efe45c287cfb8b28e6d74290d844b8bbb8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 3 Jan 2021 13:29:00 +0100
Subject: [PATCH 063/860] [twitter] try to use a Generic fallback for unknown
 twitter cards(closes #25982)

---
 youtube_dl/extractor/twitter.py | 52 +++++++++++++++++++--------------
 1 file changed, 30 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 4284487db..a35e1686c 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -251,10 +251,10 @@ class TwitterIE(TwitterBaseIE):
         'info_dict': {
             'id': '700207533655363584',
             'ext': 'mp4',
-            'title': 'simon vetugo - BEAT PROD: @suhmeduh #Damndaniel',
+            'title': 'simon vertugo - BEAT PROD: @suhmeduh #Damndaniel',
             'description': 'BEAT PROD: @suhmeduh  https://t.co/HBrQ4AfpvZ #Damndaniel https://t.co/byBooq2ejZ',
             'thumbnail': r're:^https?://.*\.jpg',
-            'uploader': 'simon vetugo',
+            'uploader': 'simon vertugo',
             'uploader_id': 'simonvertugo',
             'duration': 30.0,
             'timestamp': 1455777459,
@@ -312,6 +312,7 @@ class TwitterIE(TwitterBaseIE):
             'timestamp': 1492000653,
             'upload_date': '20170412',
         },
+        'skip': 'Account suspended',
     }, {
         'url': 'https://twitter.com/i/web/status/910031516746514432',
         'info_dict': {
@@ -380,6 +381,14 @@ class TwitterIE(TwitterBaseIE):
         # promo_video_website card
         'url': 'https://twitter.com/GunB1g/status/1163218564784017422',
         'only_matching': True,
+    }, {
+        # promo_video_convo card
+        'url': 'https://twitter.com/poco_dandy/status/1047395834013384704',
+        'only_matching': True,
+    }, {
+        # appplayer card
+        'url': 'https://twitter.com/poco_dandy/status/1150646424461176832',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -462,7 +471,25 @@ class TwitterIE(TwitterBaseIE):
                     return try_get(o, lambda x: x[x['type'].lower() + '_value'])
 
                 card_name = card['name'].split(':')[-1]
-                if card_name in ('amplify', 'promo_video_website'):
+                if card_name == 'player':
+                    info.update({
+                        '_type': 'url',
+                        'url': get_binding_value('player_url'),
+                    })
+                elif card_name == 'periscope_broadcast':
+                    info.update({
+                        '_type': 'url',
+                        'url': get_binding_value('url') or get_binding_value('player_url'),
+                        'ie_key': PeriscopeIE.ie_key(),
+                    })
+                elif card_name == 'broadcast':
+                    info.update({
+                        '_type': 'url',
+                        'url': get_binding_value('broadcast_url'),
+                        'ie_key': TwitterBroadcastIE.ie_key(),
+                    })
+                # amplify, promo_video_website, promo_video_convo, appplayer, ...
+                else:
                     is_amplify = card_name == 'amplify'
                     vmap_url = get_binding_value('amplify_url_vmap') if is_amplify else get_binding_value('player_stream_url')
                     content_id = get_binding_value('%s_content_id' % (card_name if is_amplify else 'player'))
@@ -488,25 +515,6 @@ class TwitterIE(TwitterBaseIE):
                         'duration': int_or_none(get_binding_value(
                             'content_duration_seconds')),
                     })
-                elif card_name == 'player':
-                    info.update({
-                        '_type': 'url',
-                        'url': get_binding_value('player_url'),
-                    })
-                elif card_name == 'periscope_broadcast':
-                    info.update({
-                        '_type': 'url',
-                        'url': get_binding_value('url') or get_binding_value('player_url'),
-                        'ie_key': PeriscopeIE.ie_key(),
-                    })
-                elif card_name == 'broadcast':
-                    info.update({
-                        '_type': 'url',
-                        'url': get_binding_value('broadcast_url'),
-                        'ie_key': TwitterBroadcastIE.ie_key(),
-                    })
-                else:
-                    raise ExtractorError('Unsupported Twitter Card.')
             else:
                 expanded_url = try_get(status, lambda x: x['entities']['urls'][0]['expanded_url'])
                 if not expanded_url:

From 19ec46863587758fa6de274df293ede09b10eeb3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 3 Jan 2021 13:37:24 +0100
Subject: [PATCH 064/860] [twitter] Add support for summary card(closes #25121)

---
 youtube_dl/extractor/twitter.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index a35e1686c..1190d721e 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -488,6 +488,11 @@ class TwitterIE(TwitterBaseIE):
                         'url': get_binding_value('broadcast_url'),
                         'ie_key': TwitterBroadcastIE.ie_key(),
                     })
+                elif card_name == 'summary':
+                    info.update({
+                        '_type': 'url',
+                        'url': get_binding_value('card_url'),
+                    })
                 # amplify, promo_video_website, promo_video_convo, appplayer, ...
                 else:
                     is_amplify = card_name == 'amplify'

From 0021a2b9a14626b1b56c77bec4d93e70a217d32c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 3 Jan 2021 22:25:17 +0700
Subject: [PATCH 065/860] [nrktv] Add subtitles test

---
 youtube_dl/extractor/nrk.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 69178e157..cafb85616 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -298,6 +298,14 @@ class NRKTVIE(InfoExtractor):
             'description': 'md5:46923a6e6510eefcce23d5ef2a58f2ce',
             'duration': 2223.44,
             'age_limit': 6,
+            'subtitles': {
+                'nb-nor': [{
+                    'ext': 'vtt',
+                }],
+                'nb-ttv': [{
+                    'ext': 'vtt',
+                }]
+            },
         },
     }, {
         'url': 'https://tv.nrk.no/serie/20-spoersmaal-tv/MUHH48000314/23-05-2014',

From 0889eb33e0d40d567be5b2f8431952a5517276fc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 4 Jan 2021 00:17:51 +0700
Subject: [PATCH 066/860] [xfileshare] Add support for aparat.cam (closes
 #27651)

---
 youtube_dl/extractor/xfileshare.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index 48ef07ed1..cbd5d1cbb 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -45,6 +45,7 @@ def aa_decode(aa_code):
 
 class XFileShareIE(InfoExtractor):
     _SITES = (
+        (r'aparat\.cam', 'Aparat'),
         (r'clipwatching\.com', 'ClipWatching'),
         (r'gounlimited\.to', 'GoUnlimited'),
         (r'govid\.me', 'GoVid'),
@@ -78,6 +79,9 @@ class XFileShareIE(InfoExtractor):
             'title': 'sample',
             'thumbnail': r're:http://.*\.jpg',
         },
+    }, {
+        'url': 'https://aparat.cam/n4d6dh0wvlpr',
+        'only_matching': True,
     }]
 
     @staticmethod

From e88c9ef62a4a26cc77370b741a4244d298c7d45a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 4 Jan 2021 00:51:55 +0100
Subject: [PATCH 067/860] [utils] add a function to clean podcast URLs

---
 test/test_utils.py  |  5 +++++
 youtube_dl/utils.py | 17 +++++++++++++++++
 2 files changed, 22 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index d49d3239c..259c4763e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -21,6 +21,7 @@ from youtube_dl.utils import (
     encode_base_n,
     caesar,
     clean_html,
+    clean_podcast_url,
     date_from_str,
     DateRange,
     detect_exe_version,
@@ -1470,6 +1471,10 @@ Line 1
         self.assertEqual(get_elements_by_attribute('class', 'foo', html), [])
         self.assertEqual(get_elements_by_attribute('class', 'no-such-foo', html), [])
 
+    def test_clean_podcast_url(self):
+        self.assertEqual(clean_podcast_url('https://www.podtrac.com/pts/redirect.mp3/chtbl.com/track/5899E/traffic.megaphone.fm/HSW7835899191.mp3'), 'https://traffic.megaphone.fm/HSW7835899191.mp3')
+        self.assertEqual(clean_podcast_url('https://play.podtrac.com/npr-344098539/edge1.pod.npr.org/anon.npr-podcasts/podcast/npr/waitwait/2020/10/20201003_waitwait_wwdtmpodcast201003-015621a5-f035-4eca-a9a1-7c118d90bc3c.mp3'), 'https://edge1.pod.npr.org/anon.npr-podcasts/podcast/npr/waitwait/2020/10/20201003_waitwait_wwdtmpodcast201003-015621a5-f035-4eca-a9a1-7c118d90bc3c.mp3')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d5fb6fd24..8e4d144c9 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -5706,3 +5706,20 @@ def random_birthday(year_field, month_field, day_field):
         month_field: str(random_date.month),
         day_field: str(random_date.day),
     }
+
+
+def clean_podcast_url(url):
+    return re.sub(r'''(?x)
+        (?:
+            (?:
+                chtbl\.com/track|
+                media\.blubrry\.com| # https://create.blubrry.com/resources/podcast-media-download-statistics/getting-started/
+                play\.podtrac\.com
+            )/[^/]+|
+            (?:dts|www)\.podtrac\.com/(?:pts/)?redirect\.[0-9a-z]{3,4}| # http://analytics.podtrac.com/how-to-measure
+            flex\.acast\.com|
+            pd(?:
+                cn\.co| # https://podcorn.com/analytics-prefix/
+                st\.fm # https://podsights.com/docs/
+            )/e
+        )/''', '', url)

From a563c97c5cddf55f8989ed7ea8314ef78e30107f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 4 Jan 2021 00:53:32 +0100
Subject: [PATCH 068/860] [stitcher] clean podcast URLs

---
 youtube_dl/extractor/stitcher.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/stitcher.py b/youtube_dl/extractor/stitcher.py
index 3dd0d3b5f..822782507 100644
--- a/youtube_dl/extractor/stitcher.py
+++ b/youtube_dl/extractor/stitcher.py
@@ -4,6 +4,7 @@ from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
     clean_html,
+    clean_podcast_url,
     ExtractorError,
     int_or_none,
     str_or_none,
@@ -43,7 +44,7 @@ class StitcherBaseIE(InfoExtractor):
             'title': episode['title'].strip(),
             'description': self._extract_description(episode),
             'duration': int_or_none(episode.get('duration')),
-            'url': audio_url,
+            'url': clean_podcast_url(audio_url),
             'vcodec': 'none',
             'timestamp': int_or_none(episode.get('date_published')),
             'season_number': int_or_none(episode.get('season')),

From 0e96b4b5ce79a5f5c990247f6260da40f7d71a7c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 4 Jan 2021 00:53:58 +0100
Subject: [PATCH 069/860] [acast] clean podcast URLs

---
 youtube_dl/extractor/acast.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/acast.py b/youtube_dl/extractor/acast.py
index 60378db1b..b9355a2c8 100644
--- a/youtube_dl/extractor/acast.py
+++ b/youtube_dl/extractor/acast.py
@@ -6,6 +6,7 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
+    clean_podcast_url,
     int_or_none,
     parse_iso8601,
 )
@@ -17,7 +18,7 @@ class ACastBaseIE(InfoExtractor):
         info = {
             'id': episode['id'],
             'display_id': episode.get('episodeUrl'),
-            'url': episode['url'],
+            'url': clean_podcast_url(episode['url']),
             'title': title,
             'description': clean_html(episode.get('description') or episode.get('summary')),
             'thumbnail': episode.get('image'),

From 9c484c0019ba16cd4b4d686930bfa038f1bf9320 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 4 Jan 2021 01:00:31 +0100
Subject: [PATCH 070/860] [iheart] Add new extractor for iHeartRadio(#27037)

---
 youtube_dl/extractor/extractors.py |  4 ++
 youtube_dl/extractor/iheart.py     | 97 ++++++++++++++++++++++++++++++
 2 files changed, 101 insertions(+)
 create mode 100644 youtube_dl/extractor/iheart.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d1e1e9a60..82d440a6d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -463,6 +463,10 @@ from .ign import (
     OneUPIE,
     PCMagIE,
 )
+from .iheart import (
+    IHeartRadioIE,
+    IHeartRadioPodcastIE,
+)
 from .imdb import (
     ImdbIE,
     ImdbListIE
diff --git a/youtube_dl/extractor/iheart.py b/youtube_dl/extractor/iheart.py
new file mode 100644
index 000000000..6710baeb4
--- /dev/null
+++ b/youtube_dl/extractor/iheart.py
@@ -0,0 +1,97 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_podcast_url,
+    int_or_none,
+    str_or_none,
+)
+
+
+class IHeartRadioBaseIE(InfoExtractor):
+    def _call_api(self, path, video_id, fatal=True, query=None):
+        return self._download_json(
+            'https://api.iheart.com/api/v3/podcast/' + path,
+            video_id, fatal=fatal, query=query)
+
+    def _extract_episode(self, episode):
+        return {
+            'thumbnail': episode.get('imageUrl'),
+            'description': episode.get('description'),
+            'timestamp': int_or_none(episode.get('startDate'), 1000),
+            'duration': int_or_none(episode.get('duration')),
+        }
+
+
+class IHeartRadioIE(IHeartRadioBaseIE):
+    IENAME = 'iheartradio'
+    _VALID_URL = r'(?:https?://(?:www\.)?iheart\.com/podcast/[^/]+/episode/(?P<display_id>[^/?&#]+)-|iheartradio:)(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://www.iheart.com/podcast/105-behind-the-bastards-29236323/episode/part-one-alexander-lukashenko-the-dictator-70346499/?embed=true',
+        'md5': 'c8609c92c8688dcb69d8541042b8abca',
+        'info_dict': {
+            'id': '70346499',
+            'ext': 'mp3',
+            'title': 'Part One: Alexander Lukashenko: The Dictator of Belarus',
+            'description': 'md5:66480b2d25ec93a5f60c0faa3275ce5c',
+            'timestamp': 1597741200,
+            'upload_date': '20200818',
+        }
+    }
+
+    def _real_extract(self, url):
+        episode_id = self._match_id(url)
+        episode = self._call_api(
+            'episodes/' + episode_id, episode_id)['episode']
+        info = self._extract_episode(episode)
+        print(episode['mediaUrl'])
+        info.update({
+            'id': episode_id,
+            'title': episode['title'],
+            'url': clean_podcast_url(episode['mediaUrl']),
+        })
+        return info
+
+
+class IHeartRadioPodcastIE(IHeartRadioBaseIE):
+    IE_NAME = 'iheartradio:podcast'
+    _VALID_URL = r'https?://(?:www\.)?iheart(?:podcastnetwork)?\.com/podcast/[^/?&#]+-(?P<id>\d+)/?(?:[?#&]|$)'
+    _TESTS = [{
+        'url': 'https://www.iheart.com/podcast/1119-it-could-happen-here-30717896/',
+        'info_dict': {
+            'id': '30717896',
+            'title': 'It Could Happen Here',
+            'description': 'md5:5842117412a967eb0b01f8088eb663e2',
+        },
+        'playlist_mincount': 11,
+    }, {
+        'url': 'https://www.iheartpodcastnetwork.com/podcast/105-stuff-you-should-know-26940277',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        podcast_id = self._match_id(url)
+        path = 'podcasts/' + podcast_id
+        episodes = self._call_api(
+            path + '/episodes', podcast_id, query={'limit': 1000000000})['data']
+
+        entries = []
+        for episode in episodes:
+            episode_id = str_or_none(episode.get('id'))
+            if not episode_id:
+                continue
+            info = self._extract_episode(episode)
+            info.update({
+                '_type': 'url',
+                'id': episode_id,
+                'title': episode.get('title'),
+                'url': 'iheartradio:' + episode_id,
+                'ie_key': IHeartRadioIE.ie_key(),
+            })
+            entries.append(info)
+
+        podcast = self._call_api(path, podcast_id, False) or {}
+
+        return self.playlist_result(
+            entries, podcast_id, podcast.get('title'), podcast.get('description'))

From 8487e8b98afd1b469c2b9d29ee53bd173ff9a7e0 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 4 Jan 2021 01:04:20 +0100
Subject: [PATCH 071/860] [googlepodcasts] Add new extractor

---
 youtube_dl/extractor/extractors.py     |  4 ++
 youtube_dl/extractor/googlepodcasts.py | 88 ++++++++++++++++++++++++++
 2 files changed, 92 insertions(+)
 create mode 100644 youtube_dl/extractor/googlepodcasts.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 82d440a6d..3668197b1 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -423,6 +423,10 @@ from .godtube import GodTubeIE
 from .golem import GolemIE
 from .googledrive import GoogleDriveIE
 from .googleplus import GooglePlusIE
+from .googlepodcasts import (
+    GooglePodcastsIE,
+    GooglePodcastsFeedIE,
+)
 from .googlesearch import GoogleSearchIE
 from .goshgay import GoshgayIE
 from .gputechconf import GPUTechConfIE
diff --git a/youtube_dl/extractor/googlepodcasts.py b/youtube_dl/extractor/googlepodcasts.py
new file mode 100644
index 000000000..31ad79907
--- /dev/null
+++ b/youtube_dl/extractor/googlepodcasts.py
@@ -0,0 +1,88 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_podcast_url,
+    int_or_none,
+    try_get,
+    urlencode_postdata,
+)
+
+
+class GooglePodcastsBaseIE(InfoExtractor):
+    _VALID_URL_BASE = r'https?://podcasts\.google\.com/feed/'
+
+    def _batch_execute(self, func_id, video_id, params):
+        return json.loads(self._download_json(
+            'https://podcasts.google.com/_/PodcastsUi/data/batchexecute',
+            video_id, data=urlencode_postdata({
+                'f.req': json.dumps([[[func_id, json.dumps(params), None, '1']]]),
+            }), transform_source=lambda x: self._search_regex(r'(?s)(\[.+\])', x, 'data'))[0][2])
+
+    def _extract_episode(self, episode):
+        return {
+            'id': episode[4][3],
+            'title': episode[8],
+            'url': clean_podcast_url(episode[13]),
+            'thumbnail': episode[2],
+            'description': episode[9],
+            'creator': try_get(episode, lambda x: x[14]),
+            'timestamp': int_or_none(episode[11]),
+            'duration': int_or_none(episode[12]),
+            'series': episode[1],
+        }
+
+
+class GooglePodcastsIE(GooglePodcastsBaseIE):
+    IE_NAME = 'google:podcasts'
+    _VALID_URL = GooglePodcastsBaseIE._VALID_URL_BASE + r'(?P<feed_url>[^/]+)/episode/(?P<id>[^/?&#]+)'
+    _TEST = {
+        'url': 'https://podcasts.google.com/feed/aHR0cHM6Ly9mZWVkcy5ucHIub3JnLzM0NDA5ODUzOS9wb2RjYXN0LnhtbA/episode/MzBlNWRlN2UtOWE4Yy00ODcwLTk2M2MtM2JlMmUyNmViOTRh',
+        'md5': 'fa56b2ee8bd0703e27e42d4b104c4766',
+        'info_dict': {
+            'id': '30e5de7e-9a8c-4870-963c-3be2e26eb94a',
+            'ext': 'mp3',
+            'title': 'WWDTM New Year 2021',
+            'description': 'We say goodbye to 2020 with Christine Baranksi, Doug Jones, Jonna Mendez, and Kellee Edwards.',
+            'upload_date': '20210102',
+            'timestamp': 1609606800,
+            'duration': 2901,
+            'series': "Wait Wait... Don't Tell Me!",
+        }
+    }
+
+    def _real_extract(self, url):
+        b64_feed_url, b64_guid = re.match(self._VALID_URL, url).groups()
+        episode = self._batch_execute(
+            'oNjqVe', b64_guid, [b64_feed_url, b64_guid])[1]
+        return self._extract_episode(episode)
+
+
+class GooglePodcastsFeedIE(GooglePodcastsBaseIE):
+    IE_NAME = 'google:podcasts:feed'
+    _VALID_URL = GooglePodcastsBaseIE._VALID_URL_BASE + r'(?P<id>[^/?&#]+)/?(?:[?#&]|$)'
+    _TEST = {
+        'url': 'https://podcasts.google.com/feed/aHR0cHM6Ly9mZWVkcy5ucHIub3JnLzM0NDA5ODUzOS9wb2RjYXN0LnhtbA',
+        'info_dict': {
+            'title': "Wait Wait... Don't Tell Me!",
+            'description': "NPR's weekly current events quiz. Have a laugh and test your news knowledge while figuring out what's real and what we've made up.",
+        },
+        'playlist_mincount': 20,
+    }
+
+    def _real_extract(self, url):
+        b64_feed_url = self._match_id(url)
+        data = self._batch_execute('ncqJEe', b64_feed_url, [b64_feed_url])
+
+        entries = []
+        for episode in (try_get(data, lambda x: x[1][0]) or []):
+            entries.append(self._extract_episode(episode))
+
+        feed = try_get(data, lambda x: x[3]) or []
+        return self.playlist_result(
+            entries, playlist_title=try_get(feed, lambda x: x[0]),
+            playlist_description=try_get(feed, lambda x: x[2]))

From ac61f2e0581ad15727870e8dd9a80ddacf01636e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 4 Jan 2021 01:09:11 +0100
Subject: [PATCH 072/860] [applepodcasts] Add new extractor(#25918)

---
 youtube_dl/extractor/applepodcasts.py | 61 +++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py    |  1 +
 2 files changed, 62 insertions(+)
 create mode 100644 youtube_dl/extractor/applepodcasts.py

diff --git a/youtube_dl/extractor/applepodcasts.py b/youtube_dl/extractor/applepodcasts.py
new file mode 100644
index 000000000..95758fece
--- /dev/null
+++ b/youtube_dl/extractor/applepodcasts.py
@@ -0,0 +1,61 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_podcast_url,
+    int_or_none,
+    parse_iso8601,
+    try_get,
+)
+
+
+class ApplePodcastsIE(InfoExtractor):
+    _VALID_URL = r'https?://podcasts\.apple\.com/(?:[^/]+/)?podcast(?:/[^/]+){1,2}.*?\bi=(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://podcasts.apple.com/us/podcast/207-whitney-webb-returns/id1135137367?i=1000482637777',
+        'md5': 'df02e6acb11c10e844946a39e7222b08',
+        'info_dict': {
+            'id': '1000482637777',
+            'ext': 'mp3',
+            'title': '207 - Whitney Webb Returns',
+            'description': 'md5:13a73bade02d2e43737751e3987e1399',
+            'upload_date': '20200705',
+            'timestamp': 1593921600,
+            'duration': 6425,
+            'series': 'The Tim Dillon Show',
+        }
+    }, {
+        'url': 'https://podcasts.apple.com/podcast/207-whitney-webb-returns/id1135137367?i=1000482637777',
+        'only_matching': True,
+    }, {
+        'url': 'https://podcasts.apple.com/podcast/207-whitney-webb-returns?i=1000482637777',
+        'only_matching': True,
+    }, {
+        'url': 'https://podcasts.apple.com/podcast/id1135137367?i=1000482637777',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        episode_id = self._match_id(url)
+        webpage = self._download_webpage(url, episode_id)
+        ember_data = self._parse_json(self._search_regex(
+            r'id="shoebox-ember-data-store"[^>]*>\s*({.+?})\s*<',
+            webpage, 'ember data'), episode_id)
+        episode = ember_data['data']['attributes']
+        description = episode.get('description') or {}
+
+        series = None
+        for inc in (ember_data.get('included') or []):
+            if inc.get('type') == 'media/podcast':
+                series = try_get(inc, lambda x: x['attributes']['name'])
+
+        return {
+            'id': episode_id,
+            'title': episode['name'],
+            'url': clean_podcast_url(episode['assetUrl']),
+            'description': description.get('standard') or description.get('short'),
+            'timestamp': parse_iso8601(episode.get('releaseDateTime')),
+            'duration': int_or_none(episode.get('durationInMilliseconds'), 1000),
+            'series': series,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 3668197b1..dbc8f89a6 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -55,6 +55,7 @@ from .appletrailers import (
     AppleTrailersIE,
     AppleTrailersSectionIE,
 )
+from .applepodcasts import ApplePodcastsIE
 from .archiveorg import ArchiveOrgIE
 from .arcpublishing import ArcPublishingIE
 from .arkena import ArkenaIE

From 964a8eb754dd082069419a2fda1ecedea62b7cc5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 4 Jan 2021 01:13:53 +0100
Subject: [PATCH 073/860] [googleplus] Remove Extractor(closes #4955)(closes
 #7400)

---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/googleplus.py | 73 ------------------------------
 2 files changed, 74 deletions(-)
 delete mode 100644 youtube_dl/extractor/googleplus.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index dbc8f89a6..a82f6e5f0 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -423,7 +423,6 @@ from .go import GoIE
 from .godtube import GodTubeIE
 from .golem import GolemIE
 from .googledrive import GoogleDriveIE
-from .googleplus import GooglePlusIE
 from .googlepodcasts import (
     GooglePodcastsIE,
     GooglePodcastsFeedIE,
diff --git a/youtube_dl/extractor/googleplus.py b/youtube_dl/extractor/googleplus.py
deleted file mode 100644
index 6b927bb44..000000000
--- a/youtube_dl/extractor/googleplus.py
+++ /dev/null
@@ -1,73 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-import codecs
-
-from .common import InfoExtractor
-from ..utils import unified_strdate
-
-
-class GooglePlusIE(InfoExtractor):
-    IE_DESC = 'Google Plus'
-    _VALID_URL = r'https?://plus\.google\.com/(?:[^/]+/)*?posts/(?P<id>\w+)'
-    IE_NAME = 'plus.google'
-    _TEST = {
-        'url': 'https://plus.google.com/u/0/108897254135232129896/posts/ZButuJc6CtH',
-        'info_dict': {
-            'id': 'ZButuJc6CtH',
-            'ext': 'flv',
-            'title': '嘆きの天使 降臨',
-            'upload_date': '20120613',
-            'uploader': '井上ヨシマサ',
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        # Step 1, Retrieve post webpage to extract further information
-        webpage = self._download_webpage(url, video_id, 'Downloading entry webpage')
-
-        title = self._og_search_description(webpage).splitlines()[0]
-        upload_date = unified_strdate(self._html_search_regex(
-            r'''(?x)<a.+?class="o-U-s\s[^"]+"\s+style="display:\s*none"\s*>
-                    ([0-9]{4}-[0-9]{2}-[0-9]{2})</a>''',
-            webpage, 'upload date', fatal=False, flags=re.VERBOSE))
-        uploader = self._html_search_regex(
-            r'rel="author".*?>(.*?)</a>', webpage, 'uploader', fatal=False)
-
-        # Step 2, Simulate clicking the image box to launch video
-        DOMAIN = 'https://plus.google.com/'
-        video_page = self._search_regex(
-            r'<a href="((?:%s)?photos/.*?)"' % re.escape(DOMAIN),
-            webpage, 'video page URL')
-        if not video_page.startswith(DOMAIN):
-            video_page = DOMAIN + video_page
-
-        webpage = self._download_webpage(video_page, video_id, 'Downloading video page')
-
-        def unicode_escape(s):
-            decoder = codecs.getdecoder('unicode_escape')
-            return re.sub(
-                r'\\u[0-9a-fA-F]{4,}',
-                lambda m: decoder(m.group(0))[0],
-                s)
-
-        # Extract video links all sizes
-        formats = [{
-            'url': unicode_escape(video_url),
-            'ext': 'flv',
-            'width': int(width),
-            'height': int(height),
-        } for width, height, video_url in re.findall(
-            r'\d+,(\d+),(\d+),"(https?://[^.]+\.googleusercontent\.com.*?)"', webpage)]
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'uploader': uploader,
-            'upload_date': upload_date,
-            'formats': formats,
-        }

From 83db801cbf35c130558b48f338948e0d3d3d871f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 4 Jan 2021 09:42:27 +0100
Subject: [PATCH 074/860] [iheart] remove print statement

---
 youtube_dl/extractor/iheart.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/iheart.py b/youtube_dl/extractor/iheart.py
index 6710baeb4..7a7295ff4 100644
--- a/youtube_dl/extractor/iheart.py
+++ b/youtube_dl/extractor/iheart.py
@@ -45,7 +45,6 @@ class IHeartRadioIE(IHeartRadioBaseIE):
         episode = self._call_api(
             'episodes/' + episode_id, episode_id)['episode']
         info = self._extract_episode(episode)
-        print(episode['mediaUrl'])
         info.update({
             'id': episode_id,
             'title': episode['title'],

From 745db8899d77c56bf14443be60970aed1d9e2bdd Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 4 Jan 2021 09:59:20 +0100
Subject: [PATCH 075/860] [iheart] clean HTML tags from episode description

---
 youtube_dl/extractor/iheart.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iheart.py b/youtube_dl/extractor/iheart.py
index 7a7295ff4..266c67a76 100644
--- a/youtube_dl/extractor/iheart.py
+++ b/youtube_dl/extractor/iheart.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
     clean_podcast_url,
     int_or_none,
     str_or_none,
@@ -18,7 +19,7 @@ class IHeartRadioBaseIE(InfoExtractor):
     def _extract_episode(self, episode):
         return {
             'thumbnail': episode.get('imageUrl'),
-            'description': episode.get('description'),
+            'description': clean_html(episode.get('description')),
             'timestamp': int_or_none(episode.get('startDate'), 1000),
             'duration': int_or_none(episode.get('duration')),
         }

From 8a3797a4abdc0b63b6fcbd1fbc9d81acc57ec142 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 4 Jan 2021 12:16:54 +0100
Subject: [PATCH 076/860] [nrk] fix extraction for videos without a legalAge
 rating

---
 youtube_dl/extractor/nrk.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index cafb85616..40dee2162 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -223,12 +223,12 @@ class NRKIE(NRKBaseIE):
         legal_age = try_get(
             data, lambda x: x['legalAge']['body']['rating']['code'], compat_str)
         # https://en.wikipedia.org/wiki/Norwegian_Media_Authority
-        if legal_age == 'A':
-            age_limit = 0
-        elif legal_age.isdigit():
-            age_limit = int_or_none(legal_age)
-        else:
-            age_limit = None
+        age_limit = None
+        if legal_age:
+            if legal_age == 'A':
+                age_limit = 0
+            elif legal_age.isdigit():
+                age_limit = int_or_none(legal_age)
 
         is_series = try_get(data, lambda x: x['_links']['series']['name']) == 'series'
 

From f6ea29e24bf67d13d21de3c623975891a8d61ac1 Mon Sep 17 00:00:00 2001
From: Yurii H <gardenapple@posteo.net>
Date: Mon, 4 Jan 2021 18:16:27 +0200
Subject: [PATCH 077/860] [iheart] Update test description value  (#27037)

the description has no HTML tags now.
---
 youtube_dl/extractor/iheart.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/iheart.py b/youtube_dl/extractor/iheart.py
index 266c67a76..b54c05eeb 100644
--- a/youtube_dl/extractor/iheart.py
+++ b/youtube_dl/extractor/iheart.py
@@ -35,7 +35,7 @@ class IHeartRadioIE(IHeartRadioBaseIE):
             'id': '70346499',
             'ext': 'mp3',
             'title': 'Part One: Alexander Lukashenko: The Dictator of Belarus',
-            'description': 'md5:66480b2d25ec93a5f60c0faa3275ce5c',
+            'description': 'md5:96cc7297b3a5a9ebae28643801c96fae',
             'timestamp': 1597741200,
             'upload_date': '20200818',
         }

From 766fcdd0fad8495222b2b5a14f1626960cd79d89 Mon Sep 17 00:00:00 2001
From: Kevin O'Connor <kevin@kevo.io>
Date: Mon, 4 Jan 2021 13:24:01 -0500
Subject: [PATCH 078/860] [downloader/hls] Disable decryption in tests (#27660)

Tests truncate the download to 10241 bytes, which is not divisible by 16
and cannot be decrypted. Tests don't really care about the decrypted
content, just that the data they retrieved is the expected data.
Therefore, it's fine to just return the encrypted data to tests.

See: #27621 and #27620
---
 youtube_dl/downloader/hls.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/hls.py b/youtube_dl/downloader/hls.py
index 5e1ff4f6b..7aaebc940 100644
--- a/youtube_dl/downloader/hls.py
+++ b/youtube_dl/downloader/hls.py
@@ -172,8 +172,12 @@ class HlsFD(FragmentFD):
                         iv = decrypt_info.get('IV') or compat_struct_pack('>8xq', media_sequence)
                         decrypt_info['KEY'] = decrypt_info.get('KEY') or self.ydl.urlopen(
                             self._prepare_url(info_dict, info_dict.get('_decryption_key_url') or decrypt_info['URI'])).read()
-                        frag_content = AES.new(
-                            decrypt_info['KEY'], AES.MODE_CBC, iv).decrypt(frag_content)
+                        # Don't decrypt the content in tests since the data is explicitly truncated and it's not to a valid block
+                        # size (see https://github.com/ytdl-org/youtube-dl/pull/27660). Tests only care that the correct data downloaded,
+                        # not what it decrypts to.
+                        if not test:
+                            frag_content = AES.new(
+                                decrypt_info['KEY'], AES.MODE_CBC, iv).decrypt(frag_content)
                     self._append_fragment(ctx, frag_content)
                     # We only download the first fragment during the test
                     if test:

From 9237aaa77f7e2f0b9fc17c66589423632172f473 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jan 2021 01:34:28 +0700
Subject: [PATCH 079/860] [workflows/ci.yml] Add support for jython

---
 .github/workflows/ci.yml     | 26 +++++++++++++++++++++++++-
 devscripts/install_jython.sh |  5 -----
 2 files changed, 25 insertions(+), 6 deletions(-)
 delete mode 100755 devscripts/install_jython.sh

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 99944e848..ac34196cb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -10,28 +10,52 @@ jobs:
         os: [ubuntu-latest]
         # TODO: python 2.6
         python-version: [2.7, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, pypy-2.7, pypy-3.6, pypy-3.7]
+        python-impl: [cpython]
         ytdl-test-set: [core, download]
         run-tests-ext: [sh]
         include:
         # python 3.2 is only available on windows via setup-python
         - os: windows-latest
           python-version: 3.2
+          python-impl: cpython
           ytdl-test-set: core
           run-tests-ext: bat
         - os: windows-latest
           python-version: 3.2
+          python-impl: cpython
           ytdl-test-set: download
           run-tests-ext: bat
+        # jython
+        - os: ubuntu-latest
+          python-impl: jython
+          ytdl-test-set: core
+          run-tests-ext: sh
+        - os: ubuntu-latest
+          python-impl: jython
+          ytdl-test-set: download
+          run-tests-ext: sh
     steps:
     - uses: actions/checkout@v2
     - name: Set up Python ${{ matrix.python-version }}
       uses: actions/setup-python@v2
+      if: ${{ matrix.python-impl == 'cpython' }}
       with:
         python-version: ${{ matrix.python-version }}
+    - name: Set up Java 8
+      if: ${{ matrix.python-impl == 'jython' }}
+      uses: actions/setup-java@v1
+      with:
+        java-version: 8
+    - name: Install Jython
+      if: ${{ matrix.python-impl == 'jython' }}
+      run: |
+        wget http://search.maven.org/remotecontent?filepath=org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar -O jython-installer.jar
+        java -jar jython-installer.jar -s -d "$HOME/jython"
+        echo "$HOME/jython/bin" >> $GITHUB_PATH
     - name: Install nose
       run: pip install nose
     - name: Run tests
-      continue-on-error: ${{ matrix.ytdl-test-set == 'download' }}
+      continue-on-error: ${{ matrix.ytdl-test-set == 'download' || matrix.python-impl == 'jython' }}
       env:
         YTDL_TEST_SET: ${{ matrix.ytdl-test-set }}
       run: ./devscripts/run_tests.${{ matrix.run-tests-ext }}
diff --git a/devscripts/install_jython.sh b/devscripts/install_jython.sh
deleted file mode 100755
index bafca4da4..000000000
--- a/devscripts/install_jython.sh
+++ /dev/null
@@ -1,5 +0,0 @@
-#!/bin/bash
-
-wget http://central.maven.org/maven2/org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar
-java -jar jython-installer-2.7.1.jar -s -d "$HOME/jython"
-$HOME/jython/bin/jython -m pip install nose

From c3399cac19a51897024545fb267f7579ea42318b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jan 2021 02:44:27 +0700
Subject: [PATCH 080/860] [travis] Drop Travis CI (closes #7193, closes #12366)

---
 .travis.yml | 50 --------------------------------------------------
 1 file changed, 50 deletions(-)
 delete mode 100644 .travis.yml

diff --git a/.travis.yml b/.travis.yml
deleted file mode 100644
index d828d027d..000000000
--- a/.travis.yml
+++ /dev/null
@@ -1,50 +0,0 @@
-language: python
-python:
-  - "2.6"
-  - "2.7"
-  - "3.2"
-  - "3.3"
-  - "3.4"
-  - "3.5"
-  - "3.6"
-  - "pypy"
-  - "pypy3"
-dist: trusty
-env:
-  - YTDL_TEST_SET=core
-#  - YTDL_TEST_SET=download
-jobs:
-  include:
-    - python: 3.7
-      dist: xenial
-      env: YTDL_TEST_SET=core
-#    - python: 3.7
-#      dist: xenial
-#      env: YTDL_TEST_SET=download
-    - python: 3.8
-      dist: xenial
-      env: YTDL_TEST_SET=core
-#    - python: 3.8
-#      dist: xenial
-#      env: YTDL_TEST_SET=download
-    - python: 3.8-dev
-      dist: xenial
-      env: YTDL_TEST_SET=core
-#    - python: 3.8-dev
-#      dist: xenial
-#      env: YTDL_TEST_SET=download
-    - env: JYTHON=true; YTDL_TEST_SET=core
-#    - env: JYTHON=true; YTDL_TEST_SET=download
-    - name: flake8
-      python: 3.8
-      dist: xenial
-      install: pip install flake8
-      script: flake8 .
-  fast_finish: true
-  allow_failures:
-#    - env: YTDL_TEST_SET=download
-    - env: JYTHON=true; YTDL_TEST_SET=core
-#    - env: JYTHON=true; YTDL_TEST_SET=download
-before_install:
-  - if [ "$JYTHON" == "true" ]; then ./devscripts/install_jython.sh; export PATH="$HOME/jython/bin:$PATH"; fi
-script: ./devscripts/run_tests.sh

From f318882955b90bead8206ee411641e65037b1011 Mon Sep 17 00:00:00 2001
From: cladmi <hartergaetan@gmail.com>
Date: Wed, 16 Dec 2020 09:54:48 +0100
Subject: [PATCH 081/860] [motherless] Fix recent videos upload date extraction
 (closes #27661)

Less than a week old videos use a '20h ago' or '1d ago' format.

I kept the support for 'Ago' with uppercase start at is was already in the code.
---
 youtube_dl/extractor/motherless.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index b1615b4d8..6cc36b308 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -85,18 +85,27 @@ class MotherlessIE(InfoExtractor):
             or 'http://cdn4.videos.motherlessmedia.com/videos/%s.mp4?fs=opencloud' % video_id)
         age_limit = self._rta_search(webpage)
         view_count = str_to_int(self._html_search_regex(
-            (r'>(\d+)\s+Views<', r'<strong>Views</strong>\s+([^<]+)<'),
+            (r'>([\d,.]+)\s+Views<',  # 1,234,567 Views
+             r'<strong>Views</strong>\s+([^<]+)<'),
             webpage, 'view count', fatal=False))
         like_count = str_to_int(self._html_search_regex(
-            (r'>(\d+)\s+Favorites<', r'<strong>Favorited</strong>\s+([^<]+)<'),
+            (r'>([\d,.]+)\s+Favorites<',  # 1,234 Favorites
+             r'<strong>Favorited</strong>\s+([^<]+)<'),
             webpage, 'like count', fatal=False))
 
         upload_date = self._html_search_regex(
             (r'class=["\']count[^>]+>(\d+\s+[a-zA-Z]{3}\s+\d{4})<',
+             r'class=["\']count[^>]+>(\d+[hd])\s+[aA]go<',  # 20h/1d ago
              r'<strong>Uploaded</strong>\s+([^<]+)<'), webpage, 'upload date')
-        if 'Ago' in upload_date:
-            days = int(re.search(r'([0-9]+)', upload_date).group(1))
-            upload_date = (datetime.datetime.now() - datetime.timedelta(days=days)).strftime('%Y%m%d')
+        relative = re.match(r'(\d+)([hd])$', upload_date)
+        if relative:
+            delta = int(relative.group(1))
+            unit = relative.group(2)
+            if unit == 'h':
+                delta_t = datetime.timedelta(hours=delta)
+            else:  # unit == 'd'
+                delta_t = datetime.timedelta(days=delta)
+            upload_date = (datetime.datetime.now() - delta_t).strftime('%Y%m%d')
         else:
             upload_date = unified_strdate(upload_date)
 

From ecae54a98d2a8d9300142bf3d586f31e8144ccd6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 5 Jan 2021 07:40:06 +0700
Subject: [PATCH 082/860] [motherless] Fix review issues and improve extraction
 (closes #26495, closes #27450)

---
 youtube_dl/extractor/motherless.py | 52 +++++++++++++++++++-----------
 1 file changed, 34 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index 6cc36b308..ef1e081f2 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -61,6 +61,23 @@ class MotherlessIE(InfoExtractor):
         # no keywords
         'url': 'http://motherless.com/8B4BBC1',
         'only_matching': True,
+    }, {
+        # see https://motherless.com/videos/recent for recent videos with
+        # uploaded date in "ago" format
+        'url': 'https://motherless.com/3C3E2CF',
+        'info_dict': {
+            'id': '3C3E2CF',
+            'ext': 'mp4',
+            'title': 'a/ Hot Teens',
+            'categories': list,
+            'upload_date': '20210104',
+            'uploader_id': 'yonbiw',
+            'thumbnail': r're:https?://.*\.jpg',
+            'age_limit': 18,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):
@@ -85,29 +102,28 @@ class MotherlessIE(InfoExtractor):
             or 'http://cdn4.videos.motherlessmedia.com/videos/%s.mp4?fs=opencloud' % video_id)
         age_limit = self._rta_search(webpage)
         view_count = str_to_int(self._html_search_regex(
-            (r'>([\d,.]+)\s+Views<',  # 1,234,567 Views
-             r'<strong>Views</strong>\s+([^<]+)<'),
+            (r'>([\d,.]+)\s+Views<', r'<strong>Views</strong>\s+([^<]+)<'),
             webpage, 'view count', fatal=False))
         like_count = str_to_int(self._html_search_regex(
-            (r'>([\d,.]+)\s+Favorites<',  # 1,234 Favorites
+            (r'>([\d,.]+)\s+Favorites<',
              r'<strong>Favorited</strong>\s+([^<]+)<'),
             webpage, 'like count', fatal=False))
 
-        upload_date = self._html_search_regex(
-            (r'class=["\']count[^>]+>(\d+\s+[a-zA-Z]{3}\s+\d{4})<',
-             r'class=["\']count[^>]+>(\d+[hd])\s+[aA]go<',  # 20h/1d ago
-             r'<strong>Uploaded</strong>\s+([^<]+)<'), webpage, 'upload date')
-        relative = re.match(r'(\d+)([hd])$', upload_date)
-        if relative:
-            delta = int(relative.group(1))
-            unit = relative.group(2)
-            if unit == 'h':
-                delta_t = datetime.timedelta(hours=delta)
-            else:  # unit == 'd'
-                delta_t = datetime.timedelta(days=delta)
-            upload_date = (datetime.datetime.now() - delta_t).strftime('%Y%m%d')
-        else:
-            upload_date = unified_strdate(upload_date)
+        upload_date = unified_strdate(self._search_regex(
+            r'class=["\']count[^>]+>(\d+\s+[a-zA-Z]{3}\s+\d{4})<', webpage,
+            'upload date', default=None))
+        if not upload_date:
+            uploaded_ago = self._search_regex(
+                r'>\s*(\d+[hd])\s+[aA]go\b', webpage, 'uploaded ago',
+                default=None)
+            if uploaded_ago:
+                delta = int(uploaded_ago[:-1])
+                _AGO_UNITS = {
+                    'h': 'hours',
+                    'd': 'days',
+                }
+                kwargs = {_AGO_UNITS.get(uploaded_ago[-1]): delta}
+                upload_date = (datetime.datetime.utcnow() - datetime.timedelta(**kwargs)).strftime('%Y%m%d')
 
         comment_count = webpage.count('class="media-comment-contents"')
         uploader_id = self._html_search_regex(

From be1a3f2d11b9d88c10b624b12b00b508d923983d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 5 Jan 2021 17:31:19 +0100
Subject: [PATCH 083/860] [dplay] Add suport Discovery+ domains(closes #27680)

---
 youtube_dl/extractor/dplay.py | 29 ++++++++++++++++++++++++++---
 1 file changed, 26 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index a7b9db568..47501dbe6 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -17,7 +17,12 @@ from ..utils import (
 class DPlayIE(InfoExtractor):
     _VALID_URL = r'''(?x)https?://
         (?P<domain>
-            (?:www\.)?(?P<host>dplay\.(?P<country>dk|fi|jp|se|no))|
+            (?:www\.)?(?P<host>d
+                (?:
+                    play\.(?P<country>dk|fi|jp|se|no)|
+                    iscoveryplus\.(?P<plus_country>dk|es|fi|it|se|no)
+                )
+            )|
             (?P<subdomain_country>es|it)\.dplay\.com
         )/[^/]+/(?P<id>[^/]+/[^/?#]+)'''
 
@@ -126,6 +131,24 @@ class DPlayIE(InfoExtractor):
     }, {
         'url': 'https://www.dplay.jp/video/gold-rush/24086',
         'only_matching': True,
+    }, {
+        'url': 'https://www.discoveryplus.se/videos/nugammalt-77-handelser-som-format-sverige/nugammalt-77-handelser-som-format-sverige-101',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.discoveryplus.dk/videoer/ted-bundy-mind-of-a-monster/ted-bundy-mind-of-a-monster',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.discoveryplus.no/videoer/i-kongens-klr/sesong-1-episode-7',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.discoveryplus.it/videos/biografie-imbarazzanti/luigi-di-maio-la-psicosi-di-stanislawskij',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.discoveryplus.es/videos/la-fiebre-del-oro/temporada-8-episodio-1',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.discoveryplus.fi/videot/shifting-gears-with-aaron-kaufman/episode-16',
+        'only_matching': True,
     }]
 
     def _get_disco_api_info(self, url, display_id, disco_host, realm, country):
@@ -241,7 +264,7 @@ class DPlayIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         display_id = mobj.group('id')
         domain = mobj.group('domain').lstrip('www.')
-        country = mobj.group('country') or mobj.group('subdomain_country')
-        host = 'disco-api.' + domain if domain.startswith('dplay.') else 'eu2-prod.disco-api.com'
+        country = mobj.group('country') or mobj.group('subdomain_country') or mobj.group('plus_country')
+        host = 'disco-api.' + domain if domain[0] == 'd' else 'eu2-prod.disco-api.com'
         return self._get_disco_api_info(
             url, display_id, host, 'dplay' + country, country)

From 8f757c735306355e1743a444364b79b681dea661 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 5 Jan 2021 18:17:04 +0100
Subject: [PATCH 084/860] [ketnet] fix extraction(closes #27662)

---
 youtube_dl/extractor/ketnet.py | 119 ++++++++++++++-------------------
 1 file changed, 49 insertions(+), 70 deletions(-)

diff --git a/youtube_dl/extractor/ketnet.py b/youtube_dl/extractor/ketnet.py
index 93a98e1e0..e0599d02f 100644
--- a/youtube_dl/extractor/ketnet.py
+++ b/youtube_dl/extractor/ketnet.py
@@ -2,92 +2,71 @@ from __future__ import unicode_literals
 
 from .canvas import CanvasIE
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse_unquote
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+)
 
 
 class KetnetIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ketnet\.be/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?ketnet\.be/(?P<id>(?:[^/]+/)*[^/?#&]+)'
     _TESTS = [{
-        'url': 'https://www.ketnet.be/kijken/zomerse-filmpjes',
-        'md5': '6bdeb65998930251bbd1c510750edba9',
+        'url': 'https://www.ketnet.be/kijken/n/nachtwacht/3/nachtwacht-s3a1-de-greystook',
+        'md5': '37b2b7bb9b3dcaa05b67058dc3a714a9',
         'info_dict': {
-            'id': 'zomerse-filmpjes',
+            'id': 'pbs-pub-aef8b526-115e-4006-aa24-e59ff6c6ef6f$vid-ddb815bf-c8e7-467b-8879-6bad7a32cebd',
             'ext': 'mp4',
-            'title': 'Gluur mee op de filmset en op Pennenzakkenrock',
-            'description': 'Gluur mee met Ghost Rockers op de filmset',
+            'title': 'Nachtwacht - Reeks 3: Aflevering 1',
+            'description': 'De Nachtwacht krijgt te maken met een parasiet',
             'thumbnail': r're:^https?://.*\.jpg$',
-        }
-    }, {
-        # mzid in playerConfig instead of sources
-        'url': 'https://www.ketnet.be/kijken/nachtwacht/de-greystook',
-        'md5': '90139b746a0a9bd7bb631283f6e2a64e',
-        'info_dict': {
-            'id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
-            'display_id': 'md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
-            'ext': 'flv',
-            'title': 'Nachtwacht: De Greystook',
-            'description': 'md5:1db3f5dc4c7109c821261e7512975be7',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 1468.03,
+            'duration': 1468.02,
+            'timestamp': 1609225200,
+            'upload_date': '20201229',
+            'series': 'Nachtwacht',
+            'season': 'Reeks 3',
+            'episode': 'De Greystook',
+            'episode_number': 1,
         },
         'expected_warnings': ['is not a supported codec', 'Unknown MIME type'],
     }, {
-        'url': 'https://www.ketnet.be/kijken/karrewiet/uitzending-8-september-2016',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.ketnet.be/achter-de-schermen/sien-repeteert-voor-stars-for-life',
-        'only_matching': True,
-    }, {
-        # mzsource, geo restricted to Belgium
-        'url': 'https://www.ketnet.be/kijken/nachtwacht/de-bermadoe',
+        'url': 'https://www.ketnet.be/themas/karrewiet/jaaroverzicht-20200/karrewiet-het-jaar-van-black-mamba',
         'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        display_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        video = self._download_json(
+            'https://senior-bff.ketnet.be/graphql', display_id, query={
+                'query': '''{
+  video(id: "content/ketnet/nl/%s.model.json") {
+    description
+    episodeNr
+    imageUrl
+    mediaReference
+    programTitle
+    publicationDate
+    seasonTitle
+    subtitleVideodetail
+    titleVideodetail
+  }
+}''' % display_id,
+            })['data']['video']
 
-        config = self._parse_json(
-            self._search_regex(
-                r'(?s)playerConfig\s*=\s*({.+?})\s*;', webpage,
-                'player config'),
-            video_id)
-
-        mzid = config.get('mzid')
-        if mzid:
-            return self.url_result(
-                'https://mediazone.vrt.be/api/v1/ketnet/assets/%s' % mzid,
-                CanvasIE.ie_key(), video_id=mzid)
-
-        title = config['title']
-
-        formats = []
-        for source_key in ('', 'mz'):
-            source = config.get('%ssource' % source_key)
-            if not isinstance(source, dict):
-                continue
-            for format_id, format_url in source.items():
-                if format_id == 'hls':
-                    formats.extend(self._extract_m3u8_formats(
-                        format_url, video_id, 'mp4',
-                        entry_protocol='m3u8_native', m3u8_id=format_id,
-                        fatal=False))
-                elif format_id == 'hds':
-                    formats.extend(self._extract_f4m_formats(
-                        format_url, video_id, f4m_id=format_id, fatal=False))
-                else:
-                    formats.append({
-                        'url': format_url,
-                        'format_id': format_id,
-                    })
-        self._sort_formats(formats)
+        mz_id = compat_urllib_parse_unquote(video['mediaReference'])
 
         return {
-            'id': video_id,
-            'title': title,
-            'description': config.get('description'),
-            'thumbnail': config.get('image'),
-            'series': config.get('program'),
-            'episode': config.get('episode'),
-            'formats': formats,
+            '_type': 'url_transparent',
+            'id': mz_id,
+            'title': video['titleVideodetail'],
+            'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/' + mz_id,
+            'thumbnail': video.get('imageUrl'),
+            'description': video.get('description'),
+            'timestamp': parse_iso8601(video.get('publicationDate')),
+            'series': video.get('programTitle'),
+            'season': video.get('seasonTitle'),
+            'episode': video.get('subtitleVideodetail'),
+            'episode_number': int_or_none(video.get('episodeNr')),
+            'ie_key': CanvasIE.ie_key(),
         }

From fcd90d258305cdafa5bd23c50443229205fcb9ed Mon Sep 17 00:00:00 2001
From: nixxo <nixxo@protonmail.com>
Date: Tue, 5 Jan 2021 19:49:56 +0100
Subject: [PATCH 085/860] [rai] Detect ContentItem in iframe (closes #12652)
 (#27673)

Co-authored-by: Sergey M. <dstftw@gmail.com>
---
 youtube_dl/extractor/rai.py | 16 +++++++++++++++-
 1 file changed, 15 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index ecb628f14..0a68d16b0 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -326,6 +326,19 @@ class RaiIE(RaiBaseIE):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # ContentItem in iframe (see #12652)
+        'url': 'http://www.presadiretta.rai.it/dl/portali/site/puntata/ContentItem-3ed19d13-26c2-46ff-a551-b10828262f1b.html',
+        'info_dict': {
+            'id': '1ad6dc64-444a-42a4-9bea-e5419ad2f5fd',
+            'ext': 'mp4',
+            'title': 'Partiti acchiappavoti - Presa diretta del 13/09/2015',
+            'description': 'md5:d291b03407ec505f95f27970c0b025f4',
+            'upload_date': '20150913',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         # Direct MMS URL
         'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-b63a4089-ac28-48cf-bca5-9f5b5bc46df5.html',
@@ -403,7 +416,8 @@ class RaiIE(RaiBaseIE):
                 r'''(?x)
                     (?:
                         (?:initEdizione|drawMediaRaiTV)\(|
-                        <(?:[^>]+\bdata-id|var\s+uniquename)=
+                        <(?:[^>]+\bdata-id|var\s+uniquename)=|
+                        <iframe[^>]+\bsrc=
                     )
                     (["\'])
                     (?:(?!\1).)*\bContentItem-(?P<id>%s)

From 5b24f8f505582f353c3c2f7b79b5f67612ba9c87 Mon Sep 17 00:00:00 2001
From: 23rd <23rd@vivaldi.net>
Date: Sun, 3 Jan 2021 16:30:56 +0300
Subject: [PATCH 086/860] [twitch] Switch access token to GraphQL and refactor.

---
 youtube_dl/extractor/twitch.py | 114 ++++++++++++++++++++-------------
 1 file changed, 68 insertions(+), 46 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index ab6654432..50dcb93ef 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -160,7 +160,64 @@ class TwitchBaseIE(InfoExtractor):
         return compat_str(self._parse_json(token, channel_name)['channel_id'])
 
 
-class TwitchVodIE(TwitchBaseIE):
+class TwitchGraphQLBaseIE(TwitchBaseIE):
+    _PAGE_LIMIT = 100
+
+    _OPERATION_HASHES = {
+        'CollectionSideBar': '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14',
+        'FilterableVideoTower_Videos': 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb',
+        'ClipsCards__User': 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777',
+        'ChannelCollectionsContent': '07e3691a1bad77a36aba590c351180439a40baefc1c275356f40fc7082419a84',
+        'StreamMetadata': '1c719a40e481453e5c48d9bb585d971b8b372f8ebb105b17076722264dfa5b3e',
+        'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
+        'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
+    }
+
+    def _download_base_gql(self, video_id, ops, note, fatal=True):
+        return self._download_json(
+            'https://gql.twitch.tv/gql', video_id, note,
+            data=json.dumps(ops).encode(),
+            headers={
+                'Content-Type': 'text/plain;charset=UTF-8',
+                'Client-ID': self._CLIENT_ID,
+            }, fatal=fatal)
+
+    def _download_gql(self, video_id, ops, note, fatal=True):
+        for op in ops:
+            op['extensions'] = {
+                'persistedQuery': {
+                    'version': 1,
+                    'sha256Hash': self._OPERATION_HASHES[op['operationName']],
+                }
+            }
+        return self._download_base_gql(video_id, ops, note)
+
+    def _download_access_token_gql(self, video_id, item_type=None):
+        if item_type == 'vod':
+            method = 'videoPlaybackAccessToken'
+            param_name = 'id'
+        else:
+            method = 'streamPlaybackAccessToken'
+            param_name = 'channelName'
+        ops = {
+            'query': '''{
+              %s(
+                %s: "%s",
+                  params: {
+                    platform: "web",
+                    playerBackend: "mediaplayer",
+                    playerType: "site"
+                  }) {
+                value
+                signature
+              }
+            }''' % (method, param_name, video_id),
+        }
+        note = 'Downloading access token GraphQL'
+        return self._download_base_gql(video_id, ops, note)['data'][method]
+
+
+class TwitchVodIE(TwitchGraphQLBaseIE):
     IE_NAME = 'twitch:vod'
     _VALID_URL = r'''(?x)
                     https?://
@@ -276,9 +333,7 @@ class TwitchVodIE(TwitchBaseIE):
         vod_id = self._match_id(url)
 
         info = self._download_info(vod_id)
-        access_token = self._call_api(
-            'api/vods/%s/access_token' % vod_id, vod_id,
-            'Downloading %s access token' % self._ITEM_TYPE)
+        access_token = self._download_access_token_gql(vod_id, self._ITEM_TYPE)
 
         formats = self._extract_m3u8_formats(
             '%s/vod/%s.m3u8?%s' % (
@@ -289,8 +344,8 @@ class TwitchVodIE(TwitchBaseIE):
                     'allow_spectre': 'true',
                     'player': 'twitchweb',
                     'playlist_include_framerate': 'true',
-                    'nauth': access_token['token'],
-                    'nauthsig': access_token['sig'],
+                    'nauth': access_token['value'],
+                    'nauthsig': access_token['signature'],
                 })),
             vod_id, 'mp4', entry_protocol='m3u8_native')
 
@@ -333,36 +388,6 @@ def _make_video_result(node):
     }
 
 
-class TwitchGraphQLBaseIE(TwitchBaseIE):
-    _PAGE_LIMIT = 100
-
-    _OPERATION_HASHES = {
-        'CollectionSideBar': '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14',
-        'FilterableVideoTower_Videos': 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb',
-        'ClipsCards__User': 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777',
-        'ChannelCollectionsContent': '07e3691a1bad77a36aba590c351180439a40baefc1c275356f40fc7082419a84',
-        'StreamMetadata': '1c719a40e481453e5c48d9bb585d971b8b372f8ebb105b17076722264dfa5b3e',
-        'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
-        'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
-    }
-
-    def _download_gql(self, video_id, ops, note, fatal=True):
-        for op in ops:
-            op['extensions'] = {
-                'persistedQuery': {
-                    'version': 1,
-                    'sha256Hash': self._OPERATION_HASHES[op['operationName']],
-                }
-            }
-        return self._download_json(
-            'https://gql.twitch.tv/gql', video_id, note,
-            data=json.dumps(ops).encode(),
-            headers={
-                'Content-Type': 'text/plain;charset=UTF-8',
-                'Client-ID': self._CLIENT_ID,
-            }, fatal=fatal)
-
-
 class TwitchCollectionIE(TwitchGraphQLBaseIE):
     _VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/collections/(?P<id>[^/]+)'
 
@@ -814,8 +839,8 @@ class TwitchStreamIE(TwitchGraphQLBaseIE):
         if not stream:
             raise ExtractorError('%s is offline' % channel_name, expected=True)
 
-        access_token = self._download_access_token(channel_name)
-        token = access_token['token']
+        access_token = self._download_access_token_gql(channel_name)
+        token = access_token['value']
 
         stream_id = stream.get('id') or channel_name
         query = {
@@ -826,7 +851,7 @@ class TwitchStreamIE(TwitchGraphQLBaseIE):
             'player': 'twitchweb',
             'playlist_include_framerate': 'true',
             'segment_preference': '4',
-            'sig': access_token['sig'].encode('utf-8'),
+            'sig': access_token['signature'].encode('utf-8'),
             'token': token.encode('utf-8'),
         }
         formats = self._extract_m3u8_formats(
@@ -866,7 +891,7 @@ class TwitchStreamIE(TwitchGraphQLBaseIE):
         }
 
 
-class TwitchClipsIE(TwitchBaseIE):
+class TwitchClipsIE(TwitchGraphQLBaseIE):
     IE_NAME = 'twitch:clips'
     _VALID_URL = r'''(?x)
                     https?://
@@ -912,8 +937,8 @@ class TwitchClipsIE(TwitchBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        clip = self._download_json(
-            'https://gql.twitch.tv/gql', video_id, data=json.dumps({
+        clip = self._download_base_gql(
+            video_id, {
                 'query': '''{
   clip(slug: "%s") {
     broadcaster {
@@ -937,10 +962,7 @@ class TwitchClipsIE(TwitchBaseIE):
     }
     viewCount
   }
-}''' % video_id,
-            }).encode(), headers={
-                'Client-ID': self._CLIENT_ID,
-            })['data']['clip']
+}''' % video_id}, 'Downloading clip GraphQL')['data']['clip']
 
         if not clip:
             raise ExtractorError(

From ccc71122915e630d99e8266c73a2eba26707f199 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 Jan 2021 02:10:44 +0700
Subject: [PATCH 087/860] [twitch] Improve access token extraction and remove
 unused code (closes #27646)

---
 youtube_dl/extractor/twitch.py | 33 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 50dcb93ef..a939ea24e 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -192,29 +192,27 @@ class TwitchGraphQLBaseIE(TwitchBaseIE):
             }
         return self._download_base_gql(video_id, ops, note)
 
-    def _download_access_token_gql(self, video_id, item_type=None):
-        if item_type == 'vod':
-            method = 'videoPlaybackAccessToken'
-            param_name = 'id'
-        else:
-            method = 'streamPlaybackAccessToken'
-            param_name = 'channelName'
+    def _download_access_token_gql(self, video_id, token_kind, param_name):
+        method = '%sPlaybackAccessToken' % token_kind
         ops = {
             'query': '''{
               %s(
                 %s: "%s",
-                  params: {
-                    platform: "web",
-                    playerBackend: "mediaplayer",
-                    playerType: "site"
-                  }) {
+                params: {
+                  platform: "web",
+                  playerBackend: "mediaplayer",
+                  playerType: "site"
+                }
+              )
+              {
                 value
                 signature
               }
             }''' % (method, param_name, video_id),
         }
-        note = 'Downloading access token GraphQL'
-        return self._download_base_gql(video_id, ops, note)['data'][method]
+        return self._download_base_gql(
+            video_id, ops,
+            'Downloading %s access token GraphQL' % token_kind)['data'][method]
 
 
 class TwitchVodIE(TwitchGraphQLBaseIE):
@@ -227,8 +225,6 @@ class TwitchVodIE(TwitchGraphQLBaseIE):
                         )
                         (?P<id>\d+)
                     '''
-    _ITEM_TYPE = 'vod'
-    _ITEM_SHORTCUT = 'v'
 
     _TESTS = [{
         'url': 'http://www.twitch.tv/riotgames/v/6528877?t=5m10s',
@@ -333,7 +329,7 @@ class TwitchVodIE(TwitchGraphQLBaseIE):
         vod_id = self._match_id(url)
 
         info = self._download_info(vod_id)
-        access_token = self._download_access_token_gql(vod_id, self._ITEM_TYPE)
+        access_token = self._download_access_token_gql(vod_id, 'video', 'id')
 
         formats = self._extract_m3u8_formats(
             '%s/vod/%s.m3u8?%s' % (
@@ -839,7 +835,8 @@ class TwitchStreamIE(TwitchGraphQLBaseIE):
         if not stream:
             raise ExtractorError('%s is offline' % channel_name, expected=True)
 
-        access_token = self._download_access_token_gql(channel_name)
+        access_token = self._download_access_token_gql(
+            channel_name, 'stream', 'channelName')
         token = access_token['value']
 
         stream_id = stream.get('id') or channel_name

From 1ae7ae0b969d378ea41e6b90b9c5d44358d3e36b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 5 Jan 2021 21:17:08 +0100
Subject: [PATCH 088/860] [canvas] Fix VRT NU extraction(closes #26957)(closes
 #27053)

---
 youtube_dl/extractor/canvas.py | 120 ++++++++++++---------------------
 1 file changed, 43 insertions(+), 77 deletions(-)

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index 8667a0d04..65d65d52e 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -7,12 +7,12 @@ from .common import InfoExtractor
 from .gigya import GigyaBaseIE
 from ..compat import compat_HTTPError
 from ..utils import (
+    extract_attributes,
     ExtractorError,
     strip_or_none,
     float_or_none,
     int_or_none,
     merge_dicts,
-    parse_iso8601,
     str_or_none,
     url_or_none,
 )
@@ -37,6 +37,7 @@ class CanvasIE(InfoExtractor):
         'url': 'https://mediazone.vrt.be/api/v1/canvas/assets/mz-ast-5e5f90b6-2d72-4c40-82c2-e134f884e93e',
         'only_matching': True,
     }]
+    _GEO_BYPASS = False
     _HLS_ENTRY_PROTOCOLS_MAP = {
         'HLS': 'm3u8_native',
         'HLS_AES': 'm3u8',
@@ -47,29 +48,34 @@ class CanvasIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         site_id, video_id = mobj.group('site_id'), mobj.group('id')
 
-        # Old API endpoint, serves more formats but may fail for some videos
-        data = self._download_json(
-            'https://mediazone.vrt.be/api/v1/%s/assets/%s'
-            % (site_id, video_id), video_id, 'Downloading asset JSON',
-            'Unable to download asset JSON', fatal=False)
+        data = None
+        if site_id != 'vrtvideo':
+            # Old API endpoint, serves more formats but may fail for some videos
+            data = self._download_json(
+                'https://mediazone.vrt.be/api/v1/%s/assets/%s'
+                % (site_id, video_id), video_id, 'Downloading asset JSON',
+                'Unable to download asset JSON', fatal=False)
 
         # New API endpoint
         if not data:
+            headers = self.geo_verification_headers()
+            headers.update({'Content-Type': 'application/json'})
             token = self._download_json(
                 '%s/tokens' % self._REST_API_BASE, video_id,
-                'Downloading token', data=b'',
-                headers={'Content-Type': 'application/json'})['vrtPlayerToken']
+                'Downloading token', data=b'', headers=headers)['vrtPlayerToken']
             data = self._download_json(
                 '%s/videos/%s' % (self._REST_API_BASE, video_id),
-                video_id, 'Downloading video JSON', fatal=False, query={
+                video_id, 'Downloading video JSON', query={
                     'vrtPlayerToken': token,
                     'client': '%s@PROD' % site_id,
                 }, expected_status=400)
-            message = data.get('message')
-            if message and not data.get('title'):
-                if data.get('code') == 'AUTHENTICATION_REQUIRED':
-                    self.raise_login_required(message)
-                raise ExtractorError(message, expected=True)
+            if not data.get('title'):
+                code = data.get('code')
+                if code == 'AUTHENTICATION_REQUIRED':
+                    self.raise_login_required()
+                elif code == 'INVALID_LOCATION':
+                    self.raise_geo_restricted(countries=['BE'])
+                raise ExtractorError(data.get('message') or code, expected=True)
 
         title = data['title']
         description = data.get('description')
@@ -208,17 +214,21 @@ class VrtNUIE(GigyaBaseIE):
     _VALID_URL = r'https?://(?:www\.)?vrt\.be/(?P<site_id>vrtnu)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _TESTS = [{
         # Available via old API endpoint
-        'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1/postbus-x-s1a1/',
+        'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1989/postbus-x-s1989a1/',
         'info_dict': {
-            'id': 'pbs-pub-2e2d8c27-df26-45c9-9dc6-90c78153044d$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de',
+            'id': 'pbs-pub-e8713dac-899e-41de-9313-81269f4c04ac$vid-90c932b1-e21d-4fb8-99b1-db7b49cf74de',
             'ext': 'mp4',
-            'title': 'De zwarte weduwe',
-            'description': 'md5:db1227b0f318c849ba5eab1fef895ee4',
+            'title': 'Postbus X - Aflevering 1 (Seizoen 1989)',
+            'description': 'md5:b704f669eb9262da4c55b33d7c6ed4b7',
             'duration': 1457.04,
             'thumbnail': r're:^https?://.*\.jpg$',
-            'season': 'Season 1',
-            'season_number': 1,
+            'series': 'Postbus X',
+            'season': 'Seizoen 1989',
+            'season_number': 1989,
+            'episode': 'De zwarte weduwe',
             'episode_number': 1,
+            'timestamp': 1595822400,
+            'upload_date': '20200727',
         },
         'skip': 'This video is only available for registered users',
         'params': {
@@ -300,69 +310,25 @@ class VrtNUIE(GigyaBaseIE):
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
-        webpage, urlh = self._download_webpage_handle(url, display_id)
+        webpage = self._download_webpage(url, display_id)
+
+        attrs = extract_attributes(self._search_regex(
+            r'(<nui-media[^>]+>)', webpage, 'media element'))
+        video_id = attrs['videoid']
+        publication_id = attrs.get('publicationid')
+        if publication_id:
+            video_id = publication_id + '$' + video_id
+
+        page = (self._parse_json(self._search_regex(
+            r'digitalData\s*=\s*({.+?});', webpage, 'digial data',
+            default='{}'), video_id, fatal=False) or {}).get('page') or {}
 
         info = self._search_json_ld(webpage, display_id, default={})
-
-        # title is optional here since it may be extracted by extractor
-        # that is delegated from here
-        title = strip_or_none(self._html_search_regex(
-            r'(?ms)<h1 class="content__heading">(.+?)</h1>',
-            webpage, 'title', default=None))
-
-        description = self._html_search_regex(
-            r'(?ms)<div class="content__description">(.+?)</div>',
-            webpage, 'description', default=None)
-
-        season = self._html_search_regex(
-            [r'''(?xms)<div\ class="tabs__tab\ tabs__tab--active">\s*
-                    <span>seizoen\ (.+?)</span>\s*
-                </div>''',
-             r'<option value="seizoen (\d{1,3})" data-href="[^"]+?" selected>'],
-            webpage, 'season', default=None)
-
-        season_number = int_or_none(season)
-
-        episode_number = int_or_none(self._html_search_regex(
-            r'''(?xms)<div\ class="content__episode">\s*
-                    <abbr\ title="aflevering">afl</abbr>\s*<span>(\d+)</span>
-                </div>''',
-            webpage, 'episode_number', default=None))
-
-        release_date = parse_iso8601(self._html_search_regex(
-            r'(?ms)<div class="content__broadcastdate">\s*<time\ datetime="(.+?)"',
-            webpage, 'release_date', default=None))
-
-        # If there's a ? or a # in the URL, remove them and everything after
-        clean_url = urlh.geturl().split('?')[0].split('#')[0].strip('/')
-        securevideo_url = clean_url + '.mssecurevideo.json'
-
-        try:
-            video = self._download_json(securevideo_url, display_id)
-        except ExtractorError as e:
-            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
-                self.raise_login_required()
-            raise
-
-        # We are dealing with a '../<show>.relevant' URL
-        redirect_url = video.get('url')
-        if redirect_url:
-            return self.url_result(self._proto_relative_url(redirect_url, 'https:'))
-
-        # There is only one entry, but with an unknown key, so just get
-        # the first one
-        video_id = list(video.values())[0].get('videoid')
-
         return merge_dicts(info, {
             '_type': 'url_transparent',
             'url': 'https://mediazone.vrt.be/api/v1/vrtvideo/assets/%s' % video_id,
             'ie_key': CanvasIE.ie_key(),
             'id': video_id,
             'display_id': display_id,
-            'title': title,
-            'description': description,
-            'season': season,
-            'season_number': season_number,
-            'episode_number': episode_number,
-            'release_date': release_date,
+            'season_number': int_or_none(page.get('episode_season')),
         })

From 799c794947045e04b920be5ca9fc037cdfe42d3c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 Jan 2021 03:33:45 +0700
Subject: [PATCH 089/860] [twitch:vod] Switch to GraphQL for video metadata

---
 youtube_dl/extractor/twitch.py | 43 ++++++++++++++++++++++++++++++++--
 1 file changed, 41 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index a939ea24e..d4491cc42 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -171,6 +171,7 @@ class TwitchGraphQLBaseIE(TwitchBaseIE):
         'StreamMetadata': '1c719a40e481453e5c48d9bb585d971b8b372f8ebb105b17076722264dfa5b3e',
         'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
         'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
+        'VideoMetadata': '226edb3e692509f727fd56821f5653c05740242c82b0388883e0c0e75dcbf687',
     }
 
     def _download_base_gql(self, video_id, ops, note, fatal=True):
@@ -234,7 +235,7 @@ class TwitchVodIE(TwitchGraphQLBaseIE):
             'title': 'LCK Summer Split - Week 6 Day 1',
             'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 17208,
-            'timestamp': 1435131709,
+            'timestamp': 1435131734,
             'upload_date': '20150624',
             'uploader': 'Riot Games',
             'uploader_id': 'riotgames',
@@ -288,6 +289,22 @@ class TwitchVodIE(TwitchGraphQLBaseIE):
                 'kraken/videos/%s' % item_id, item_id,
                 'Downloading video info JSON'))
 
+    def _download_info_gql(self, item_id):
+        data = self._download_gql(
+            item_id, [{
+                'operationName': 'VideoMetadata',
+                'variables': {
+                    'channelLogin': '',
+                    'videoID': item_id,
+                },
+            }],
+            'Downloading stream metadata GraphQL')[0]['data']
+        video = data.get('video')
+        if video is None:
+            raise ExtractorError(
+                'Video %s does not exist' % item_id, expected=True)
+        return self._extract_info_gql(video, item_id)
+
     @staticmethod
     def _extract_info(info):
         status = info.get('status')
@@ -325,10 +342,32 @@ class TwitchVodIE(TwitchGraphQLBaseIE):
             'is_live': is_live,
         }
 
+    @staticmethod
+    def _extract_info_gql(info, item_id):
+        vod_id = info.get('id') or item_id
+        # id backward compatibility for download archives
+        if vod_id[0] != 'v':
+            vod_id = 'v%s' % vod_id
+        thumbnail = url_or_none(info.get('previewThumbnailURL'))
+        if thumbnail:
+            for p in ('width', 'height'):
+                thumbnail = thumbnail.replace('{%s}' % p, '0')
+        return {
+            'id': vod_id,
+            'title': info.get('title') or 'Untitled Broadcast',
+            'description': info.get('description'),
+            'duration': int_or_none(info.get('lengthSeconds')),
+            'thumbnail': thumbnail,
+            'uploader': try_get(info, lambda x: x['owner']['displayName'], compat_str),
+            'uploader_id': try_get(info, lambda x: x['owner']['login'], compat_str),
+            'timestamp': unified_timestamp(info.get('publishedAt')),
+            'view_count': int_or_none(info.get('viewCount')),
+        }
+
     def _real_extract(self, url):
         vod_id = self._match_id(url)
 
-        info = self._download_info(vod_id)
+        info = self._download_info_gql(vod_id)
         access_token = self._download_access_token_gql(vod_id, 'video', 'id')
 
         formats = self._extract_m3u8_formats(

From 8f20ad36dc6cadc9d2a2269bc68d1a00be82f64e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 Jan 2021 03:46:03 +0700
Subject: [PATCH 090/860] [twitch] Drop legacy kraken API v5 code altogether

---
 youtube_dl/extractor/twitch.py | 83 ----------------------------------
 1 file changed, 83 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index d4491cc42..17131c411 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -9,7 +9,6 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_kwargs,
     compat_parse_qs,
     compat_str,
     compat_urlparse,
@@ -42,31 +41,6 @@ class TwitchBaseIE(InfoExtractor):
     _CLIENT_ID = 'kimne78kx3ncx6brgo4mv6wki5h1ko'
     _NETRC_MACHINE = 'twitch'
 
-    def _handle_error(self, response):
-        if not isinstance(response, dict):
-            return
-        error = response.get('error')
-        if error:
-            raise ExtractorError(
-                '%s returned error: %s - %s' % (self.IE_NAME, error, response.get('message')),
-                expected=True)
-
-    def _call_api(self, path, item_id, *args, **kwargs):
-        headers = kwargs.get('headers', {}).copy()
-        headers.update({
-            'Accept': 'application/vnd.twitchtv.v5+json; charset=UTF-8',
-            'Client-ID': self._CLIENT_ID,
-        })
-        kwargs.update({
-            'headers': headers,
-            'expected_status': (400, 410),
-        })
-        response = self._download_json(
-            '%s/%s' % (self._API_BASE, path), item_id,
-            *args, **compat_kwargs(kwargs))
-        self._handle_error(response)
-        return response
-
     def _real_initialize(self):
         self._login()
 
@@ -151,14 +125,6 @@ class TwitchBaseIE(InfoExtractor):
                     })
         self._sort_formats(formats)
 
-    def _download_access_token(self, channel_name):
-        return self._call_api(
-            'api/channels/%s/access_token' % channel_name, channel_name,
-            'Downloading access token JSON')
-
-    def _extract_channel_id(self, token, channel_name):
-        return compat_str(self._parse_json(token, channel_name)['channel_id'])
-
 
 class TwitchGraphQLBaseIE(TwitchBaseIE):
     _PAGE_LIMIT = 100
@@ -283,12 +249,6 @@ class TwitchVodIE(TwitchGraphQLBaseIE):
         'only_matching': True,
     }]
 
-    def _download_info(self, item_id):
-        return self._extract_info(
-            self._call_api(
-                'kraken/videos/%s' % item_id, item_id,
-                'Downloading video info JSON'))
-
     def _download_info_gql(self, item_id):
         data = self._download_gql(
             item_id, [{
@@ -500,49 +460,6 @@ class TwitchPlaylistBaseIE(TwitchGraphQLBaseIE):
             if not cursor or not isinstance(cursor, compat_str):
                 break
 
-    # Deprecated kraken v5 API
-    def _entries_kraken(self, channel_name, broadcast_type, sort):
-        access_token = self._download_access_token(channel_name)
-        channel_id = self._extract_channel_id(access_token['token'], channel_name)
-        offset = 0
-        counter_override = None
-        for counter in itertools.count(1):
-            response = self._call_api(
-                'kraken/channels/%s/videos/' % channel_id,
-                channel_id,
-                'Downloading video JSON page %s' % (counter_override or counter),
-                query={
-                    'offset': offset,
-                    'limit': self._PAGE_LIMIT,
-                    'broadcast_type': broadcast_type,
-                    'sort': sort,
-                })
-            videos = response.get('videos')
-            if not isinstance(videos, list):
-                break
-            for video in videos:
-                if not isinstance(video, dict):
-                    continue
-                video_url = url_or_none(video.get('url'))
-                if not video_url:
-                    continue
-                yield {
-                    '_type': 'url_transparent',
-                    'ie_key': TwitchVodIE.ie_key(),
-                    'id': video.get('_id'),
-                    'url': video_url,
-                    'title': video.get('title'),
-                    'description': video.get('description'),
-                    'timestamp': unified_timestamp(video.get('published_at')),
-                    'duration': float_or_none(video.get('length')),
-                    'view_count': int_or_none(video.get('views')),
-                    'language': video.get('language'),
-                }
-            offset += self._PAGE_LIMIT
-            total = int_or_none(response.get('_total'))
-            if total and offset >= total:
-                break
-
 
 class TwitchVideosIE(TwitchPlaylistBaseIE):
     _VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/(?P<id>[^/]+)/(?:videos|profile)'

From 4ae243fc6ca41e4f7684db96dfdf58aaed33d100 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 6 Jan 2021 03:54:33 +0700
Subject: [PATCH 091/860] [twitch] Refactor

---
 youtube_dl/extractor/twitch.py | 48 ++++++++++++++++------------------
 1 file changed, 23 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 17131c411..db264e8a1 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -41,6 +41,17 @@ class TwitchBaseIE(InfoExtractor):
     _CLIENT_ID = 'kimne78kx3ncx6brgo4mv6wki5h1ko'
     _NETRC_MACHINE = 'twitch'
 
+    _OPERATION_HASHES = {
+        'CollectionSideBar': '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14',
+        'FilterableVideoTower_Videos': 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb',
+        'ClipsCards__User': 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777',
+        'ChannelCollectionsContent': '07e3691a1bad77a36aba590c351180439a40baefc1c275356f40fc7082419a84',
+        'StreamMetadata': '1c719a40e481453e5c48d9bb585d971b8b372f8ebb105b17076722264dfa5b3e',
+        'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
+        'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
+        'VideoMetadata': '226edb3e692509f727fd56821f5653c05740242c82b0388883e0c0e75dcbf687',
+    }
+
     def _real_initialize(self):
         self._login()
 
@@ -125,21 +136,6 @@ class TwitchBaseIE(InfoExtractor):
                     })
         self._sort_formats(formats)
 
-
-class TwitchGraphQLBaseIE(TwitchBaseIE):
-    _PAGE_LIMIT = 100
-
-    _OPERATION_HASHES = {
-        'CollectionSideBar': '27111f1b382effad0b6def325caef1909c733fe6a4fbabf54f8d491ef2cf2f14',
-        'FilterableVideoTower_Videos': 'a937f1d22e269e39a03b509f65a7490f9fc247d7f83d6ac1421523e3b68042cb',
-        'ClipsCards__User': 'b73ad2bfaecfd30a9e6c28fada15bd97032c83ec77a0440766a56fe0bd632777',
-        'ChannelCollectionsContent': '07e3691a1bad77a36aba590c351180439a40baefc1c275356f40fc7082419a84',
-        'StreamMetadata': '1c719a40e481453e5c48d9bb585d971b8b372f8ebb105b17076722264dfa5b3e',
-        'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
-        'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
-        'VideoMetadata': '226edb3e692509f727fd56821f5653c05740242c82b0388883e0c0e75dcbf687',
-    }
-
     def _download_base_gql(self, video_id, ops, note, fatal=True):
         return self._download_json(
             'https://gql.twitch.tv/gql', video_id, note,
@@ -159,7 +155,7 @@ class TwitchGraphQLBaseIE(TwitchBaseIE):
             }
         return self._download_base_gql(video_id, ops, note)
 
-    def _download_access_token_gql(self, video_id, token_kind, param_name):
+    def _download_access_token(self, video_id, token_kind, param_name):
         method = '%sPlaybackAccessToken' % token_kind
         ops = {
             'query': '''{
@@ -182,7 +178,7 @@ class TwitchGraphQLBaseIE(TwitchBaseIE):
             'Downloading %s access token GraphQL' % token_kind)['data'][method]
 
 
-class TwitchVodIE(TwitchGraphQLBaseIE):
+class TwitchVodIE(TwitchBaseIE):
     IE_NAME = 'twitch:vod'
     _VALID_URL = r'''(?x)
                     https?://
@@ -249,7 +245,7 @@ class TwitchVodIE(TwitchGraphQLBaseIE):
         'only_matching': True,
     }]
 
-    def _download_info_gql(self, item_id):
+    def _download_info(self, item_id):
         data = self._download_gql(
             item_id, [{
                 'operationName': 'VideoMetadata',
@@ -327,8 +323,8 @@ class TwitchVodIE(TwitchGraphQLBaseIE):
     def _real_extract(self, url):
         vod_id = self._match_id(url)
 
-        info = self._download_info_gql(vod_id)
-        access_token = self._download_access_token_gql(vod_id, 'video', 'id')
+        info = self._download_info(vod_id)
+        access_token = self._download_access_token(vod_id, 'video', 'id')
 
         formats = self._extract_m3u8_formats(
             '%s/vod/%s.m3u8?%s' % (
@@ -383,7 +379,7 @@ def _make_video_result(node):
     }
 
 
-class TwitchCollectionIE(TwitchGraphQLBaseIE):
+class TwitchCollectionIE(TwitchBaseIE):
     _VALID_URL = r'https?://(?:(?:www|go|m)\.)?twitch\.tv/collections/(?P<id>[^/]+)'
 
     _TESTS = [{
@@ -420,7 +416,9 @@ class TwitchCollectionIE(TwitchGraphQLBaseIE):
             entries, playlist_id=collection_id, playlist_title=title)
 
 
-class TwitchPlaylistBaseIE(TwitchGraphQLBaseIE):
+class TwitchPlaylistBaseIE(TwitchBaseIE):
+    _PAGE_LIMIT = 100
+
     def _entries(self, channel_name, *args):
         cursor = None
         variables_common = self._make_variables(channel_name, *args)
@@ -701,7 +699,7 @@ class TwitchVideosCollectionsIE(TwitchPlaylistBaseIE):
             playlist_title='%s - Collections' % channel_name)
 
 
-class TwitchStreamIE(TwitchGraphQLBaseIE):
+class TwitchStreamIE(TwitchBaseIE):
     IE_NAME = 'twitch:stream'
     _VALID_URL = r'''(?x)
                     https?://
@@ -791,7 +789,7 @@ class TwitchStreamIE(TwitchGraphQLBaseIE):
         if not stream:
             raise ExtractorError('%s is offline' % channel_name, expected=True)
 
-        access_token = self._download_access_token_gql(
+        access_token = self._download_access_token(
             channel_name, 'stream', 'channelName')
         token = access_token['value']
 
@@ -844,7 +842,7 @@ class TwitchStreamIE(TwitchGraphQLBaseIE):
         }
 
 
-class TwitchClipsIE(TwitchGraphQLBaseIE):
+class TwitchClipsIE(TwitchBaseIE):
     IE_NAME = 'twitch:clips'
     _VALID_URL = r'''(?x)
                     https?://

From 430c4bc9d02b49e180309dc55c62b96d5cada535 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 6 Jan 2021 13:41:07 +0100
Subject: [PATCH 092/860] [sbs] Add support for ondemand play and news embed
 URLs(closes #17650)(closes #27629)

---
 youtube_dl/extractor/sbs.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index 0e623ff7b..f722528cd 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 class SBSIE(InfoExtractor):
     IE_DESC = 'sbs.com.au'
-    _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/(?:ondemand|news)/video/(?:single/)?(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/(?:ondemand(?:/video/(?:single/)?|.*?\bplay=)|news/(?:embeds/)?video/)(?P<id>[0-9]+)'
 
     _TESTS = [{
         # Original URL is handled by the generic IE which finds the iframe:
@@ -18,7 +18,7 @@ class SBSIE(InfoExtractor):
         'url': 'http://www.sbs.com.au/ondemand/video/single/320403011771/?source=drupal&vertical=thefeed',
         'md5': '3150cf278965eeabb5b4cea1c963fe0a',
         'info_dict': {
-            'id': '320403011771',
+            'id': '_rFBPRPO4pMR',
             'ext': 'mp4',
             'title': 'Dingo Conservation (The Feed)',
             'description': 'md5:f250a9856fca50d22dec0b5b8015f8a5',
@@ -34,6 +34,15 @@ class SBSIE(InfoExtractor):
     }, {
         'url': 'http://www.sbs.com.au/news/video/471395907773/The-Feed-July-9',
         'only_matching': True,
+    }, {
+        'url': 'https://www.sbs.com.au/ondemand/?play=1836638787723',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.sbs.com.au/ondemand/program/inside-windsor-castle?play=1283505731842',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.sbs.com.au/news/embeds/video/1840778819866',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From b13f29098fa8c7cf8da1867661713cbedb32e431 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 6 Jan 2021 18:50:02 +0100
Subject: [PATCH 093/860] [bfmtv] Add new extractor(closes #16053)(closes
 #26615)

---
 youtube_dl/extractor/bfmtv.py      | 103 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |   5 ++
 2 files changed, 108 insertions(+)
 create mode 100644 youtube_dl/extractor/bfmtv.py

diff --git a/youtube_dl/extractor/bfmtv.py b/youtube_dl/extractor/bfmtv.py
new file mode 100644
index 000000000..501f69d80
--- /dev/null
+++ b/youtube_dl/extractor/bfmtv.py
@@ -0,0 +1,103 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import extract_attributes
+
+
+class BFMTVBaseIE(InfoExtractor):
+    _VALID_URL_BASE = r'https?://(?:www\.)?bfmtv\.com/'
+    _VALID_URL_TMPL = _VALID_URL_BASE + r'(?:[^/]+/)*[^/?&#]+_%s[A-Z]-(?P<id>\d{12})\.html'
+    _VIDEO_BLOCK_REGEX = r'(<div[^>]+class="video_block"[^>]*>)'
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s'
+
+    def _brightcove_url_result(self, video_id, video_block):
+        account_id = video_block.get('accountid') or '876450612001'
+        player_id = video_block.get('playerid') or 'I2qBTln4u'
+        return self.url_result(
+            self.BRIGHTCOVE_URL_TEMPLATE % (account_id, player_id, video_id),
+            'BrightcoveNew', video_id)
+
+
+class BFMTVIE(BFMTVBaseIE):
+    IE_NAME = 'bfmtv'
+    _VALID_URL = BFMTVBaseIE._VALID_URL_TMPL % 'V'
+    _TESTS = [{
+        'url': 'https://www.bfmtv.com/politique/emmanuel-macron-l-islam-est-une-religion-qui-vit-une-crise-aujourd-hui-partout-dans-le-monde_VN-202010020146.html',
+        'info_dict': {
+            'id': '6196747868001',
+            'ext': 'mp4',
+            'title': 'Emmanuel Macron: "L\'Islam est une religion qui vit une crise aujourd’hui, partout dans le monde"',
+            'description': 'Le Président s\'exprime sur la question du séparatisme depuis les Mureaux, dans les Yvelines.',
+            'uploader_id': '876450610001',
+            'upload_date': '20201002',
+            'timestamp': 1601629620,
+        },
+    }]
+
+    def _real_extract(self, url):
+        bfmtv_id = self._match_id(url)
+        webpage = self._download_webpage(url, bfmtv_id)
+        video_block = extract_attributes(self._search_regex(
+            self._VIDEO_BLOCK_REGEX, webpage, 'video block'))
+        return self._brightcove_url_result(video_block['videoid'], video_block)
+
+
+class BFMTVLiveIE(BFMTVIE):
+    IE_NAME = 'bfmtv:live'
+    _VALID_URL = BFMTVBaseIE._VALID_URL_BASE + '(?P<id>(?:[^/]+/)?en-direct)'
+    _TESTS = [{
+        'url': 'https://www.bfmtv.com/en-direct/',
+        'info_dict': {
+            'id': '5615950982001',
+            'ext': 'mp4',
+            'title': r're:^le direct BFMTV WEB \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
+            'uploader_id': '876450610001',
+            'upload_date': '20171018',
+            'timestamp': 1508329950,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://www.bfmtv.com/economie/en-direct/',
+        'only_matching': True,
+    }]
+
+
+class BFMTVArticleIE(BFMTVBaseIE):
+    IE_NAME = 'bfmtv:article'
+    _VALID_URL = BFMTVBaseIE._VALID_URL_TMPL % 'A'
+    _TESTS = [{
+        'url': 'https://www.bfmtv.com/sante/covid-19-un-responsable-de-l-institut-pasteur-se-demande-quand-la-france-va-se-reconfiner_AV-202101060198.html',
+        'info_dict': {
+            'id': '202101060198',
+            'title': 'Covid-19: un responsable de l\'Institut Pasteur se demande "quand la France va se reconfiner"',
+            'description': 'md5:947974089c303d3ac6196670ae262843',
+        },
+        'playlist_count': 2,
+    }, {
+        'url': 'https://www.bfmtv.com/international/pour-bolsonaro-le-bresil-est-en-faillite-mais-il-ne-peut-rien-faire_AD-202101060232.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.bfmtv.com/sante/covid-19-oui-le-vaccin-de-pfizer-distribue-en-france-a-bien-ete-teste-sur-des-personnes-agees_AN-202101060275.html',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        bfmtv_id = self._match_id(url)
+        webpage = self._download_webpage(url, bfmtv_id)
+
+        entries = []
+        for video_block_el in re.findall(self._VIDEO_BLOCK_REGEX, webpage):
+            video_block = extract_attributes(video_block_el)
+            video_id = video_block.get('videoid')
+            if not video_id:
+                continue
+            entries.append(self._brightcove_url_result(video_id, video_block))
+
+        return self.playlist_result(
+            entries, bfmtv_id, self._og_search_title(webpage, fatal=False),
+            self._html_search_meta(['og:description', 'description'], webpage))
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index a82f6e5f0..b11385685 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -101,6 +101,11 @@ from .bellmedia import BellMediaIE
 from .beatport import BeatportIE
 from .bet import BetIE
 from .bfi import BFIPlayerIE
+from .bfmtv import (
+    BFMTVIE,
+    BFMTVLiveIE,
+    BFMTVArticleIE,
+)
 from .bigflix import BigflixIE
 from .bild import BildIE
 from .bilibili import (

From a29e340efa60ecb05d85e9c6d87ec709ba26822f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 6 Jan 2021 20:38:02 +0100
Subject: [PATCH 094/860] [bibeltv] Add new extractor(closes #14361)

---
 youtube_dl/extractor/bibeltv.py | 30 ++++++++++++++++++++++++++++++
 1 file changed, 30 insertions(+)
 create mode 100644 youtube_dl/extractor/bibeltv.py

diff --git a/youtube_dl/extractor/bibeltv.py b/youtube_dl/extractor/bibeltv.py
new file mode 100644
index 000000000..56c2bfee8
--- /dev/null
+++ b/youtube_dl/extractor/bibeltv.py
@@ -0,0 +1,30 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class BibelTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?bibeltv\.de/mediathek/videos/(?:crn/)?(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.bibeltv.de/mediathek/videos/329703-sprachkurs-in-malaiisch',
+        'md5': '252f908192d611de038b8504b08bf97f',
+        'info_dict': {
+            'id': 'ref:329703',
+            'ext': 'mp4',
+            'title': 'Sprachkurs in Malaiisch',
+            'description': 'md5:3e9f197d29ee164714e67351cf737dfe',
+            'timestamp': 1608316701,
+            'uploader_id': '5840105145001',
+            'upload_date': '20201218',
+        }
+    }, {
+        'url': 'https://www.bibeltv.de/mediathek/videos/crn/326374',
+        'only_matching': True,
+    }]
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/5840105145001/default_default/index.html?videoId=ref:%s'
+
+    def _real_extract(self, url):
+        crn_id = self._match_id(url)
+        return self.url_result(
+            self.BRIGHTCOVE_URL_TEMPLATE % crn_id, 'BrightcoveNew')

From 8f29b2dd38213a4b45064fee16e64af29338a6e5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 6 Jan 2021 20:39:04 +0100
Subject: [PATCH 095/860] [extractors] add BibelTVIE import

---
 youtube_dl/extractor/extractors.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b11385685..9b449937d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -106,6 +106,7 @@ from .bfmtv import (
     BFMTVLiveIE,
     BFMTVArticleIE,
 )
+from .bibeltv import BibelTVIE
 from .bigflix import BigflixIE
 from .bild import BildIE
 from .bilibili import (

From 3d8e32dcc0bc6014fe05f445d755907ae046d5b5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 7 Jan 2021 12:35:04 +0100
Subject: [PATCH 096/860] [canvas] Match only supported VRT NU URLs(#27707)

---
 youtube_dl/extractor/canvas.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index 65d65d52e..8b76a0200 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -211,7 +211,7 @@ class CanvasEenIE(InfoExtractor):
 
 class VrtNUIE(GigyaBaseIE):
     IE_DESC = 'VrtNU.be'
-    _VALID_URL = r'https?://(?:www\.)?vrt\.be/(?P<site_id>vrtnu)/(?:[^/]+/)*(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?vrt\.be/vrtnu/a-z/(?:[^/]+/){2}(?P<id>[^/?#&]+)'
     _TESTS = [{
         # Available via old API endpoint
         'url': 'https://www.vrt.be/vrtnu/a-z/postbus-x/1989/postbus-x-s1989a1/',

From 3a61e6d36003b6043dd53dec5ae714a839225044 Mon Sep 17 00:00:00 2001
From: nixxo <nixxo@protonmail.com>
Date: Thu, 7 Jan 2021 14:48:45 +0100
Subject: [PATCH 097/860] [rai] improve subtitles extraction (#27705)

closes #27698
---
 test/test_subtitles.py      | 12 +++++++--
 youtube_dl/extractor/rai.py | 50 ++++++++++++++++++++++++-------------
 2 files changed, 43 insertions(+), 19 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 17aaaf20d..550e0ca00 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -258,16 +258,24 @@ class TestNRKSubtitles(BaseTestSubtitles):
 
 
 class TestRaiPlaySubtitles(BaseTestSubtitles):
-    url = 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html'
     IE = RaiPlayIE
 
-    def test_allsubtitles(self):
+    def test_subtitles_key(self):
+        self.url = 'http://www.raiplay.it/video/2014/04/Report-del-07042014-cb27157f-9dd0-4aee-b788-b1f67643a391.html'
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['it']))
         self.assertEqual(md5(subtitles['it']), 'b1d90a98755126b61e667567a1f6680a')
 
+    def test_subtitles_array_key(self):
+        self.url = 'https://www.raiplay.it/video/2020/12/Report---04-01-2021-2e90f1de-8eee-4de4-ac0e-78d21db5b600.html'
+        self.DL.params['writesubtitles'] = True
+        self.DL.params['allsubtitles'] = True
+        subtitles = self.getSubtitles()
+        self.assertEqual(set(subtitles.keys()), set(['it']))
+        self.assertEqual(md5(subtitles['it']), '4b3264186fbb103508abe5311cfcb9cd')
+
 
 class TestVikiSubtitles(BaseTestSubtitles):
     url = 'http://www.viki.com/videos/1060846v-punch-episode-18'
diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 0a68d16b0..67b86fc72 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -103,22 +103,28 @@ class RaiBaseIE(InfoExtractor):
         }.items() if v is not None)
 
     @staticmethod
-    def _extract_subtitles(url, subtitle_url):
+    def _extract_subtitles(url, video_data):
+        STL_EXT = 'stl'
+        SRT_EXT = 'srt'
         subtitles = {}
-        if subtitle_url and isinstance(subtitle_url, compat_str):
-            subtitle_url = urljoin(url, subtitle_url)
-            STL_EXT = '.stl'
-            SRT_EXT = '.srt'
-            subtitles['it'] = [{
-                'ext': 'stl',
-                'url': subtitle_url,
-            }]
-            if subtitle_url.endswith(STL_EXT):
-                srt_url = subtitle_url[:-len(STL_EXT)] + SRT_EXT
-                subtitles['it'].append({
-                    'ext': 'srt',
-                    'url': srt_url,
+        subtitles_array = video_data.get('subtitlesArray') or []
+        for k in ('subtitles', 'subtitlesUrl'):
+            subtitles_array.append({'url': video_data.get(k)})
+        for subtitle in subtitles_array:
+            sub_url = subtitle.get('url')
+            if sub_url and isinstance(sub_url, compat_str):
+                sub_lang = subtitle.get('language') or 'it'
+                sub_url = urljoin(url, sub_url)
+                sub_ext = determine_ext(sub_url, SRT_EXT)
+                subtitles.setdefault(sub_lang, []).append({
+                    'ext': sub_ext,
+                    'url': sub_url,
                 })
+                if STL_EXT == sub_ext:
+                    subtitles[sub_lang].append({
+                        'ext': SRT_EXT,
+                        'url': sub_url[:-len(STL_EXT)] + SRT_EXT,
+                    })
         return subtitles
 
 
@@ -138,6 +144,9 @@ class RaiPlayIE(RaiBaseIE):
             'duration': 6160,
             'series': 'Report',
             'season': '2013/14',
+            'subtitles': {
+                'it': 'count:2',
+            },
         },
         'params': {
             'skip_download': True,
@@ -145,6 +154,10 @@ class RaiPlayIE(RaiBaseIE):
     }, {
         'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?',
         'only_matching': True,
+    }, {
+        # subtitles at 'subtitlesArray' key (see #27698)
+        'url': 'https://www.raiplay.it/video/2020/12/Report---04-01-2021-2e90f1de-8eee-4de4-ac0e-78d21db5b600.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -172,7 +185,7 @@ class RaiPlayIE(RaiBaseIE):
         if date_published and time_published:
             date_published += ' ' + time_published
 
-        subtitles = self._extract_subtitles(url, video.get('subtitles'))
+        subtitles = self._extract_subtitles(url, video)
 
         program_info = media.get('program_info') or {}
         season = media.get('season')
@@ -327,7 +340,7 @@ class RaiIE(RaiBaseIE):
             'skip_download': True,
         },
     }, {
-        # ContentItem in iframe (see #12652)
+        # ContentItem in iframe (see #12652) and subtitle at 'subtitlesUrl' key
         'url': 'http://www.presadiretta.rai.it/dl/portali/site/puntata/ContentItem-3ed19d13-26c2-46ff-a551-b10828262f1b.html',
         'info_dict': {
             'id': '1ad6dc64-444a-42a4-9bea-e5419ad2f5fd',
@@ -335,6 +348,9 @@ class RaiIE(RaiBaseIE):
             'title': 'Partiti acchiappavoti - Presa diretta del 13/09/2015',
             'description': 'md5:d291b03407ec505f95f27970c0b025f4',
             'upload_date': '20150913',
+            'subtitles': {
+                'it': 'count:2',
+            },
         },
         'params': {
             'skip_download': True,
@@ -379,7 +395,7 @@ class RaiIE(RaiBaseIE):
                     'url': compat_urlparse.urljoin(url, thumbnail_url),
                 })
 
-        subtitles = self._extract_subtitles(url, media.get('subtitlesUrl'))
+        subtitles = self._extract_subtitles(url, media)
 
         info = {
             'id': content_id,

From 38a967c98ea98524e24a36ca4a9f1d1e891f7887 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Jan 2021 11:01:29 +0700
Subject: [PATCH 098/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 35 +++++++++++++++++++++++++++++++++++
 1 file changed, 35 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 9942ee578..ac4f62e59 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,38 @@
+version <unreleased>
+
+Core
+* [downloader/hls] Disable decryption in tests (#27660)
++ [utils] Add a function to clean podcast URLs
+
+Extractors
+* [rai] Improve subtitles extraction (#27698, #27705)
+* [canvas] Match only supported VRT NU URLs (#27707)
++ [bibeltv] Add support for bibeltv.de (#14361)
++ [bfmtv] Add support for bfmtv.com (#16053, #26615)
++ [sbs] Add support for ondemand play and news embed URLs (#17650, #27629)
+* [twitch] Drop legacy kraken API v5 code altogether and refactor
+* [twitch:vod] Switch to GraphQL for video metadata
+* [canvas] Fix VRT NU extraction (#26957, #27053)
+* [twitch] Switch access token to GraphQL and refactor (#27646)
++ [rai] Detect ContentItem in iframe (#12652, #27673)
+* [ketnet] Fix extraction (#27662)
++ [dplay] Add suport Discovery+ domains (#27680)
+* [motherless] Improve extraction (#26495, #27450)
+* [motherless] Fix recent videos upload date extraction (#27661)
+* [nrk] Fix extraction for videos without a legalAge rating
+- [googleplus] Remove extractor (#4955, #7400)
++ [applepodcasts] Add support for podcasts.apple.com (#25918)
++ [googlepodcasts] Add support for podcasts.google.com
++ [iheart] Add support for iheart.com (#27037)
+* [acast] Clean podcast URLs
+* [stitcher] Clean podcast URLs
++ [xfileshare] Add support for aparat.cam (#27651)
++ [twitter] Add support for summary card (#25121)
+* [twitter] Try to use a Generic fallback for unknown twitter cards (#25982)
++ [stitcher] Add support for shows and show metadata extraction (#20510)
+* [stv] Improve episode id extraction (#23083)
+
+
 version 2021.01.03
 
 Extractors

From bf6a74c620bd4d5726503c5302906bb36b009026 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 8 Jan 2021 11:03:34 +0700
Subject: [PATCH 099/860] release 2021.01.08

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          |  6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md |  4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md |  4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           |  6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      |  4 ++--
 ChangeLog                                        |  2 +-
 docs/supportedsites.md                           | 13 +++++++++++--
 youtube_dl/version.py                            |  2 +-
 8 files changed, 25 insertions(+), 16 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index ee2da644b..768d45fc1 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.08. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.01.03**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.08**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.01.03
+ [debug] youtube-dl version 2021.01.08
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index 7d535b6d1..2bd90da57 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.08. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.01.03**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.08**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 74674ce62..272895b47 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.08. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.01.03**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.08**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index 7b0402d41..608fcfba4 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.08. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.01.03**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.08**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.01.03
+ [debug] youtube-dl version 2021.01.08
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index 17ffff465..d085ab1ef 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.08. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.01.03**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.08**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index ac4f62e59..3629c4fb8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.01.08
 
 Core
 * [downloader/hls] Disable decryption in tests (#27660)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 8178af5b6..3a49043fa 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -54,6 +54,7 @@
  - **Aparat**
  - **AppleConnect**
  - **AppleDaily**: 臺灣蘋果日報
+ - **ApplePodcasts**
  - **appletrailers**
  - **appletrailers:section**
  - **archive.org**: archive.org videos
@@ -98,6 +99,10 @@
  - **BellMedia**
  - **Bet**
  - **bfi:player**
+ - **bfmtv**
+ - **bfmtv:article**
+ - **bfmtv:live**
+ - **BibelTV**
  - **Bigflix**
  - **Bild**: Bild.de
  - **BiliBili**
@@ -336,6 +341,8 @@
  - **Go**
  - **GodTube**
  - **Golem**
+ - **google:podcasts**
+ - **google:podcasts:feed**
  - **GoogleDrive**
  - **Goshgay**
  - **GPUTechConf**
@@ -370,6 +377,8 @@
  - **HungamaSong**
  - **Hypem**
  - **ign.com**
+ - **IHeartRadio**
+ - **iheartradio:podcast**
  - **imdb**: Internet Movie Database trailers
  - **imdb:list**: Internet Movie Database lists
  - **Imgur**
@@ -691,7 +700,6 @@
  - **Playwire**
  - **pluralsight**
  - **pluralsight:course**
- - **plus.google**: Google Plus
  - **podomatic**
  - **Pokemon**
  - **PolskieRadio**
@@ -862,6 +870,7 @@
  - **stanfordoc**: Stanford Open ClassRoom
  - **Steam**
  - **Stitcher**
+ - **StitcherShow**
  - **Streamable**
  - **streamcloud.eu**
  - **StreamCZ**
@@ -1123,7 +1132,7 @@
  - **WWE**
  - **XBef**
  - **XboxClips**
- - **XFileShare**: XFileShare based sites: ClipWatching, GoUnlimited, GoVid, HolaVid, Streamty, TheVideoBee, Uqload, VidBom, vidlo, VidLocker, VidShare, VUp, XVideoSharing
+ - **XFileShare**: XFileShare based sites: Aparat, ClipWatching, GoUnlimited, GoVid, HolaVid, Streamty, TheVideoBee, Uqload, VidBom, vidlo, VidLocker, VidShare, VUp, XVideoSharing
  - **XHamster**
  - **XHamsterEmbed**
  - **XHamsterUser**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 1588ee8e4..0d9659b2b 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.01.03'
+__version__ = '2021.01.08'

From 2c337f4e854f8bdf33f30d26b1a242b00a0eff93 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 8 Jan 2021 09:25:00 +0100
Subject: [PATCH 100/860] [spike] Update Paramount Network feed URL(closes
 #27715)

---
 youtube_dl/extractor/spike.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index aabff7a3c..4c5e3f7c2 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -40,9 +40,15 @@ class ParamountNetworkIE(MTVServicesInfoExtractor):
         },
     }]
 
-    _FEED_URL = 'http://www.paramountnetwork.com/feeds/mrss/'
+    _FEED_URL = 'http://feeds.mtvnservices.com/od/feed/intl-mrss-player-feed'
     _GEO_COUNTRIES = ['US']
 
+    def _get_feed_query(self, uri):
+        return {
+            'arcEp': 'paramountnetwork.com',
+            'mgid': uri,
+        }
+
     def _extract_mgid(self, webpage):
         root_data = self._parse_json(self._search_regex(
             r'window\.__DATA__\s*=\s*({.+})',

From 61e669acff712175362bea01d42d7a154d300289 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 8 Jan 2021 16:13:22 +0100
Subject: [PATCH 101/860] [khanacademy] fix extraction(closes #2887)(closes
 #26803)

---
 youtube_dl/extractor/extractors.py  |   5 +-
 youtube_dl/extractor/khanacademy.py | 137 ++++++++++++++++------------
 2 files changed, 85 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9b449937d..57d4d319c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -526,7 +526,10 @@ from .karaoketv import KaraoketvIE
 from .karrierevideos import KarriereVideosIE
 from .keezmovies import KeezMoviesIE
 from .ketnet import KetnetIE
-from .khanacademy import KhanAcademyIE
+from .khanacademy import (
+    KhanAcademyIE,
+    KhanAcademyUnitIE,
+)
 from .kickstarter import KickStarterIE
 from .kinja import KinjaEmbedIE
 from .kinopoisk import KinoPoiskIE
diff --git a/youtube_dl/extractor/khanacademy.py b/youtube_dl/extractor/khanacademy.py
index 61739efa7..87e520378 100644
--- a/youtube_dl/extractor/khanacademy.py
+++ b/youtube_dl/extractor/khanacademy.py
@@ -1,82 +1,107 @@
 from __future__ import unicode_literals
 
-import re
+import json
 
 from .common import InfoExtractor
 from ..utils import (
-    unified_strdate,
+    int_or_none,
+    parse_iso8601,
+    try_get,
 )
 
 
-class KhanAcademyIE(InfoExtractor):
-    _VALID_URL = r'^https?://(?:(?:www|api)\.)?khanacademy\.org/(?P<key>[^/]+)/(?:[^/]+/){,2}(?P<id>[^?#/]+)(?:$|[?#])'
-    IE_NAME = 'KhanAcademy'
+class KhanAcademyBaseIE(InfoExtractor):
+    _VALID_URL_TEMPL = r'https?://(?:www\.)?khanacademy\.org/(?P<id>(?:[^/]+/){%s}%s[^?#/&]+)'
 
-    _TESTS = [{
-        'url': 'http://www.khanacademy.org/video/one-time-pad',
-        'md5': '7b391cce85e758fb94f763ddc1bbb979',
+    def _parse_video(self, video):
+        return {
+            '_type': 'url_transparent',
+            'url': video['youtubeId'],
+            'id': video.get('slug'),
+            'title': video.get('title'),
+            'thumbnail': video.get('imageUrl') or video.get('thumbnailUrl'),
+            'duration': int_or_none(video.get('duration')),
+            'description': video.get('description'),
+            'ie_key': 'Youtube',
+        }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        component_props = self._parse_json(self._download_json(
+            'https://www.khanacademy.org/api/internal/graphql',
+            display_id, query={
+                'hash': 1604303425,
+                'variables': json.dumps({
+                    'path': display_id,
+                    'queryParams': '',
+                }),
+            })['data']['contentJson'], display_id)['componentProps']
+        return self._parse_component_props(component_props)
+
+
+class KhanAcademyIE(KhanAcademyBaseIE):
+    IE_NAME = 'khanacademy'
+    _VALID_URL = KhanAcademyBaseIE._VALID_URL_TEMPL % ('4', 'v/')
+    _TEST = {
+        'url': 'https://www.khanacademy.org/computing/computer-science/cryptography/crypt/v/one-time-pad',
+        'md5': '9c84b7b06f9ebb80d22a5c8dedefb9a0',
         'info_dict': {
-            'id': 'one-time-pad',
-            'ext': 'webm',
+            'id': 'FlIG3TvQCBQ',
+            'ext': 'mp4',
             'title': 'The one-time pad',
             'description': 'The perfect cipher',
             'duration': 176,
             'uploader': 'Brit Cruise',
             'uploader_id': 'khanacademy',
             'upload_date': '20120411',
+            'timestamp': 1334170113,
+            'license': 'cc-by-nc-sa',
         },
         'add_ie': ['Youtube'],
-    }, {
-        'url': 'https://www.khanacademy.org/math/applied-math/cryptography',
+    }
+
+    def _parse_component_props(self, component_props):
+        video = component_props['tutorialPageData']['contentModel']
+        info = self._parse_video(video)
+        author_names = video.get('authorNames')
+        info.update({
+            'uploader': ', '.join(author_names) if author_names else None,
+            'timestamp': parse_iso8601(video.get('dateAdded')),
+            'license': video.get('kaUserLicense'),
+        })
+        return info
+
+
+class KhanAcademyUnitIE(KhanAcademyBaseIE):
+    IE_NAME = 'khanacademy:unit'
+    _VALID_URL = (KhanAcademyBaseIE._VALID_URL_TEMPL % ('2', '')) + '/?(?:[?#&]|$)'
+    _TEST = {
+        'url': 'https://www.khanacademy.org/computing/computer-science/cryptography',
         'info_dict': {
             'id': 'cryptography',
-            'title': 'Journey into cryptography',
+            'title': 'Cryptography',
             'description': 'How have humans protected their secret messages through history? What has changed today?',
         },
-        'playlist_mincount': 3,
-    }]
+        'playlist_mincount': 31,
+    }
 
-    def _real_extract(self, url):
-        m = re.match(self._VALID_URL, url)
-        video_id = m.group('id')
+    def _parse_component_props(self, component_props):
+        curation = component_props['curation']
 
-        if m.group('key') == 'video':
-            data = self._download_json(
-                'http://api.khanacademy.org/api/v1/videos/' + video_id,
-                video_id, 'Downloading video info')
-
-            upload_date = unified_strdate(data['date_added'])
-            uploader = ', '.join(data['author_names'])
-            return {
-                '_type': 'url_transparent',
-                'url': data['url'],
-                'id': video_id,
-                'title': data['title'],
-                'thumbnail': data['image_url'],
-                'duration': data['duration'],
-                'description': data['description'],
-                'uploader': uploader,
-                'upload_date': upload_date,
+        entries = []
+        tutorials = try_get(curation, lambda x: x['tabs'][0]['modules'][0]['tutorials'], list) or []
+        for tutorial_number, tutorial in enumerate(tutorials, 1):
+            chapter_info = {
+                'chapter': tutorial.get('title'),
+                'chapter_number': tutorial_number,
+                'chapter_id': tutorial.get('id'),
             }
-        else:
-            # topic
-            data = self._download_json(
-                'http://api.khanacademy.org/api/v1/topic/' + video_id,
-                video_id, 'Downloading topic info')
+            for content_item in (tutorial.get('contentItems') or []):
+                if content_item.get('kind') == 'Video':
+                    info = self._parse_video(content_item)
+                    info.update(chapter_info)
+                    entries.append(info)
 
-            entries = [
-                {
-                    '_type': 'url',
-                    'url': c['url'],
-                    'id': c['id'],
-                    'title': c['title'],
-                }
-                for c in data['children'] if c['kind'] in ('Video', 'Topic')]
-
-            return {
-                '_type': 'playlist',
-                'id': video_id,
-                'title': data['title'],
-                'description': data['description'],
-                'entries': entries,
-            }
+        return self.playlist_result(
+            entries, curation.get('unit'), curation.get('title'),
+            curation.get('description'))

From 170e1c19951ce6a87ce0e9157faa32b9a2efe708 Mon Sep 17 00:00:00 2001
From: 0l-l0 <49962426+0l-l0@users.noreply.github.com>
Date: Fri, 8 Jan 2021 21:09:38 +0100
Subject: [PATCH 102/860] [peertube] Extract files also from streamingPlaylists
 (#27728)

JSON objects with an empty "files" tag seem to be a valid PeerTube API
response. In those cases the "files" arrays contained in the
"streamingPlaylists" members can be used instead.
closes #26002
closes #27586
---
 youtube_dl/extractor/peertube.py | 22 +++++++++++++++++++++-
 1 file changed, 21 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/peertube.py b/youtube_dl/extractor/peertube.py
index c39d12728..c2ca71c71 100644
--- a/youtube_dl/extractor/peertube.py
+++ b/youtube_dl/extractor/peertube.py
@@ -450,6 +450,18 @@ class PeerTubeIE(InfoExtractor):
             'tags': ['framasoft', 'peertube'],
             'categories': ['Science & Technology'],
         }
+    }, {
+        # Issue #26002
+        'url': 'peertube:spacepub.space:d8943b2d-8280-497b-85ec-bc282ec2afdc',
+        'info_dict': {
+            'id': 'd8943b2d-8280-497b-85ec-bc282ec2afdc',
+            'ext': 'mp4',
+            'title': 'Dot matrix printer shell demo',
+            'uploader_id': '3',
+            'timestamp': 1587401293,
+            'upload_date': '20200420',
+            'uploader': 'Drew DeVault',
+        }
     }, {
         'url': 'https://peertube.tamanoir.foucry.net/videos/watch/0b04f13d-1e18-4f1d-814e-4979aa7c9c44',
         'only_matching': True,
@@ -526,7 +538,15 @@ class PeerTubeIE(InfoExtractor):
         title = video['name']
 
         formats = []
-        for file_ in video['files']:
+        files = video.get('files') or []
+        for playlist in (video.get('streamingPlaylists') or []):
+            if not isinstance(playlist, dict):
+                continue
+            playlist_files = playlist.get('files')
+            if not (playlist_files and isinstance(playlist_files, list)):
+                continue
+            files.extend(playlist_files)
+        for file_ in files:
             if not isinstance(file_, dict):
                 continue
             file_url = url_or_none(file_.get('fileUrl'))

From ff1e76540075d41899b18bde792aef4f8ba19ffa Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 9 Jan 2021 22:29:58 +0100
Subject: [PATCH 103/860] [threeqsdn] Improve extraction(closes #21058)

---
 youtube_dl/extractor/threeqsdn.py | 162 +++++++++++++++++-------------
 1 file changed, 92 insertions(+), 70 deletions(-)

diff --git a/youtube_dl/extractor/threeqsdn.py b/youtube_dl/extractor/threeqsdn.py
index f26937da1..f6d37bb9e 100644
--- a/youtube_dl/extractor/threeqsdn.py
+++ b/youtube_dl/extractor/threeqsdn.py
@@ -3,10 +3,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_HTTPError
 from ..utils import (
     determine_ext,
-    js_to_json,
-    mimetype2ext,
+    ExtractorError,
+    float_or_none,
+    int_or_none,
+    parse_iso8601,
 )
 
 
@@ -15,29 +18,35 @@ class ThreeQSDNIE(InfoExtractor):
     IE_DESC = '3Q SDN'
     _VALID_URL = r'https?://playout\.3qsdn\.com/(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
     _TESTS = [{
-        # ondemand from http://www.philharmonie.tv/veranstaltung/26/
-        'url': 'http://playout.3qsdn.com/0280d6b9-1215-11e6-b427-0cc47a188158?protocol=http',
-        'md5': 'ab040e37bcfa2e0c079f92cb1dd7f6cd',
+        # https://player.3qsdn.com/demo.html
+        'url': 'https://playout.3qsdn.com/7201c779-6b3c-11e7-a40e-002590c750be',
+        'md5': '64a57396b16fa011b15e0ea60edce918',
         'info_dict': {
-            'id': '0280d6b9-1215-11e6-b427-0cc47a188158',
+            'id': '7201c779-6b3c-11e7-a40e-002590c750be',
             'ext': 'mp4',
-            'title': '0280d6b9-1215-11e6-b427-0cc47a188158',
+            'title': 'Video Ads',
             'is_live': False,
+            'description': 'Video Ads Demo',
+            'timestamp': 1500334803,
+            'upload_date': '20170717',
+            'duration': 888.032,
+            'subtitles': {
+                'eng': 'count:1',
+            },
         },
-        'expected_warnings': ['Failed to download MPD manifest', 'Failed to parse JSON'],
+        'expected_warnings': ['Unknown MIME type application/mp4 in DASH manifest'],
     }, {
         # live video stream
-        'url': 'https://playout.3qsdn.com/d755d94b-4ab9-11e3-9162-0025907ad44f?js=true',
+        'url': 'https://playout.3qsdn.com/66e68995-11ca-11e8-9273-002590c750be',
         'info_dict': {
-            'id': 'd755d94b-4ab9-11e3-9162-0025907ad44f',
+            'id': '66e68995-11ca-11e8-9273-002590c750be',
             'ext': 'mp4',
-            'title': 're:^d755d94b-4ab9-11e3-9162-0025907ad44f [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'title': 're:^66e68995-11ca-11e8-9273-002590c750be [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'is_live': True,
         },
         'params': {
             'skip_download': True,  # m3u8 downloads
         },
-        'expected_warnings': ['Failed to download MPD manifest'],
     }, {
         # live audio stream
         'url': 'http://playout.3qsdn.com/9edf36e0-6bf2-11e2-a16a-9acf09e2db48',
@@ -58,6 +67,14 @@ class ThreeQSDNIE(InfoExtractor):
         # live video with rtmp link
         'url': 'https://playout.3qsdn.com/6092bb9e-8f72-11e4-a173-002590c750be',
         'only_matching': True,
+    }, {
+        # ondemand from http://www.philharmonie.tv/veranstaltung/26/
+        'url': 'http://playout.3qsdn.com/0280d6b9-1215-11e6-b427-0cc47a188158?protocol=http',
+        'only_matching': True,
+    }, {
+        # live video stream
+        'url': 'https://playout.3qsdn.com/d755d94b-4ab9-11e3-9162-0025907ad44f?js=true',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -70,73 +87,78 @@ class ThreeQSDNIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        js = self._download_webpage(
-            'http://playout.3qsdn.com/%s' % video_id, video_id,
-            query={'js': 'true'})
+        try:
+            config = self._download_json(
+                url.replace('://playout.3qsdn.com/', '://playout.3qsdn.com/config/'), video_id)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+                self.raise_geo_restricted()
+            raise
 
-        if any(p in js for p in (
-                '>This content is not available in your country',
-                'playout.3qsdn.com/forbidden')):
-            self.raise_geo_restricted()
-
-        stream_content = self._search_regex(
-            r'streamContent\s*:\s*(["\'])(?P<content>.+?)\1', js,
-            'stream content', default='demand', group='content')
-
-        live = stream_content == 'live'
-
-        stream_type = self._search_regex(
-            r'streamType\s*:\s*(["\'])(?P<type>audio|video)\1', js,
-            'stream type', default='video', group='type')
+        live = config.get('streamContent') == 'live'
+        aspect = float_or_none(config.get('aspect'))
 
         formats = []
-        urls = set()
-
-        def extract_formats(item_url, item={}):
-            if not item_url or item_url in urls:
-                return
-            urls.add(item_url)
-            ext = mimetype2ext(item.get('type')) or determine_ext(item_url, default_ext=None)
-            if ext == 'mpd':
-                formats.extend(self._extract_mpd_formats(
-                    item_url, video_id, mpd_id='mpd', fatal=False))
-            elif ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    item_url, video_id, 'mp4',
-                    entry_protocol='m3u8' if live else 'm3u8_native',
-                    m3u8_id='hls', fatal=False))
-            elif ext == 'f4m':
-                formats.extend(self._extract_f4m_formats(
-                    item_url, video_id, f4m_id='hds', fatal=False))
-            else:
-                if not self._is_valid_url(item_url, video_id):
-                    return
-                formats.append({
-                    'url': item_url,
-                    'format_id': item.get('quality'),
-                    'ext': 'mp4' if item_url.startswith('rtsp') else ext,
-                    'vcodec': 'none' if stream_type == 'audio' else None,
-                })
-
-        for item_js in re.findall(r'({[^{]*?\b(?:src|source)\s*:\s*["\'].+?})', js):
-            f = self._parse_json(
-                item_js, video_id, transform_source=js_to_json, fatal=False)
-            if not f:
+        for source_type, source in (config.get('sources') or {}).items():
+            if not source:
                 continue
-            extract_formats(f.get('src'), f)
+            if source_type == 'dash':
+                formats.extend(self._extract_mpd_formats(
+                    source, video_id, mpd_id='mpd', fatal=False))
+            elif source_type == 'hls':
+                formats.extend(self._extract_m3u8_formats(
+                    source, video_id, 'mp4', 'm3u8' if live else 'm3u8_native',
+                    m3u8_id='hls', fatal=False))
+            elif source_type == 'progressive':
+                for s in source:
+                    src = s.get('src')
+                    if not (src and self._is_valid_url(src, video_id)):
+                        continue
+                    width = None
+                    format_id = ['http']
+                    ext = determine_ext(src)
+                    if ext:
+                        format_id.append(ext)
+                    height = int_or_none(s.get('height'))
+                    if height:
+                        format_id.append('%dp' % height)
+                        if aspect:
+                            width = int(height * aspect)
+                    formats.append({
+                        'ext': ext,
+                        'format_id': '-'.join(format_id),
+                        'height': height,
+                        'source_preference': 0,
+                        'url': src,
+                        'vcodec': 'none' if height == 0 else None,
+                        'width': width,
+                    })
+        for f in formats:
+            if f.get('acodec') == 'none':
+                f['preference'] = -40
+            elif f.get('vcodec') == 'none':
+                f['preference'] = -50
+        self._sort_formats(formats, ('preference', 'width', 'height', 'source_preference', 'tbr', 'vbr', 'abr', 'ext', 'format_id'))
 
-        # More relaxed version to collect additional URLs and acting
-        # as a future-proof fallback
-        for _, src in re.findall(r'\b(?:src|source)\s*:\s*(["\'])((?:https?|rtsp)://.+?)\1', js):
-            extract_formats(src)
+        subtitles = {}
+        for subtitle in (config.get('subtitles') or []):
+            src = subtitle.get('src')
+            if not src:
+                continue
+            subtitles.setdefault(subtitle.get('label') or 'eng', []).append({
+                'url': src,
+            })
 
-        self._sort_formats(formats)
-
-        title = self._live_title(video_id) if live else video_id
+        title = config.get('title') or video_id
 
         return {
             'id': video_id,
-            'title': title,
+            'title': self._live_title(title) if live else title,
+            'thumbnail': config.get('poster') or None,
+            'description': config.get('description') or None,
+            'timestamp': parse_iso8601(config.get('upload_date')),
+            'duration': float_or_none(config.get('vlength')) or None,
             'is_live': live,
             'formats': formats,
+            'subtitles': subtitles,
         }

From c2d06aef600b2af557d367b9846d79720a3d6725 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Jan 2021 06:23:08 +0700
Subject: [PATCH 104/860] [twitch] Fix authentication (refs #27743)

---
 youtube_dl/extractor/twitch.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index db264e8a1..fa19e9a7a 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -76,7 +76,7 @@ class TwitchBaseIE(InfoExtractor):
 
             headers = {
                 'Referer': page_url,
-                'Origin': page_url,
+                'Origin': 'https://www.twitch.tv',
                 'Content-Type': 'text/plain;charset=UTF-8',
             }
 

From 70f572585df5740da90feb518a93f10bb479cca2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 10 Jan 2021 06:23:58 +0700
Subject: [PATCH 105/860] [twitch] Improve login error extraction

---
 youtube_dl/extractor/twitch.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index fa19e9a7a..c4fd2f686 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -17,6 +17,7 @@ from ..compat import (
 )
 from ..utils import (
     clean_html,
+    dict_get,
     ExtractorError,
     float_or_none,
     int_or_none,
@@ -83,7 +84,7 @@ class TwitchBaseIE(InfoExtractor):
             response = self._download_json(
                 post_url, None, note, data=json.dumps(form).encode(),
                 headers=headers, expected_status=400)
-            error = response.get('error_description') or response.get('error_code')
+            error = dict_get(response, ('error', 'error_description', 'error_code'))
             if error:
                 fail(error)
 

From d0fc289f45763ab556936c2ce3d76fc1c3dd7c69 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 11 Jan 2021 01:30:54 +0700
Subject: [PATCH 106/860] [workflows/ci.yml] Change ubuntu-latest to
 ubuntu-18.04

ubuntu-latest alias will be switched to ubuntu-20.04 soon (see [1]) which lacks python 3.3 and 3.4 (see [2]).

1. https://github.blog/changelog/2020-10-29-github-actions-ubuntu-latest-workflows-will-use-ubuntu-20-04/
2. https://raw.githubusercontent.com/actions/python-versions/main/versions-manifest.json
---
 .github/workflows/ci.yml | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ac34196cb..bac3256e7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -7,7 +7,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: [ubuntu-latest]
+        os: [ubuntu-18.04]
         # TODO: python 2.6
         python-version: [2.7, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, pypy-2.7, pypy-3.6, pypy-3.7]
         python-impl: [cpython]
@@ -26,11 +26,11 @@ jobs:
           ytdl-test-set: download
           run-tests-ext: bat
         # jython
-        - os: ubuntu-latest
+        - os: ubuntu-18.04
           python-impl: jython
           ytdl-test-set: core
           run-tests-ext: sh
-        - os: ubuntu-latest
+        - os: ubuntu-18.04
           python-impl: jython
           ytdl-test-set: download
           run-tests-ext: sh

From 4759543f6e5d532795eb1d5434692bb6d5e1f0ec Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 11 Jan 2021 17:35:13 +0100
Subject: [PATCH 107/860] [youtube:search] fix view_count and try to extract
 all video sections(closes #27588)(closes #27604)

---
 youtube_dl/extractor/youtube.py | 117 ++++++++++++++------------------
 1 file changed, 50 insertions(+), 67 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 87bdc1677..f57099f8c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -308,6 +308,36 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
                 default='{}'), video_id, fatal=False)
 
+    def _extract_video(self, renderer):
+        video_id = renderer['videoId']
+        title = try_get(
+            renderer,
+            (lambda x: x['title']['runs'][0]['text'],
+             lambda x: x['title']['simpleText']), compat_str)
+        description = try_get(
+            renderer, lambda x: x['descriptionSnippet']['runs'][0]['text'],
+            compat_str)
+        duration = parse_duration(try_get(
+            renderer, lambda x: x['lengthText']['simpleText'], compat_str))
+        view_count_text = try_get(
+            renderer, lambda x: x['viewCountText']['simpleText'], compat_str) or ''
+        view_count = str_to_int(self._search_regex(
+            r'^([\d,]+)', re.sub(r'\s', '', view_count_text),
+            'view count', default=None))
+        uploader = try_get(
+            renderer, lambda x: x['ownerText']['runs'][0]['text'], compat_str)
+        return {
+            '_type': 'url_transparent',
+            'ie_key': YoutubeIE.ie_key(),
+            'id': video_id,
+            'url': video_id,
+            'title': title,
+            'description': description,
+            'duration': duration,
+            'view_count': view_count,
+            'uploader': uploader,
+        }
+
 
 class YoutubeIE(YoutubeBaseInfoExtractor):
     IE_DESC = 'YouTube.com'
@@ -2765,36 +2795,6 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             if renderer:
                 return renderer
 
-    def _extract_video(self, renderer):
-        video_id = renderer.get('videoId')
-        title = try_get(
-            renderer,
-            (lambda x: x['title']['runs'][0]['text'],
-             lambda x: x['title']['simpleText']), compat_str)
-        description = try_get(
-            renderer, lambda x: x['descriptionSnippet']['runs'][0]['text'],
-            compat_str)
-        duration = parse_duration(try_get(
-            renderer, lambda x: x['lengthText']['simpleText'], compat_str))
-        view_count_text = try_get(
-            renderer, lambda x: x['viewCountText']['simpleText'], compat_str) or ''
-        view_count = str_to_int(self._search_regex(
-            r'^([\d,]+)', re.sub(r'\s', '', view_count_text),
-            'view count', default=None))
-        uploader = try_get(
-            renderer, lambda x: x['ownerText']['runs'][0]['text'], compat_str)
-        return {
-            '_type': 'url_transparent',
-            'ie_key': YoutubeIE.ie_key(),
-            'id': video_id,
-            'url': video_id,
-            'title': title,
-            'description': description,
-            'duration': duration,
-            'view_count': view_count,
-            'uploader': uploader,
-        }
-
     def _grid_entries(self, grid_renderer):
         for item in grid_renderer['items']:
             if not isinstance(item, dict):
@@ -3417,46 +3417,29 @@ class YoutubeSearchIE(SearchInfoExtractor, YoutubeBaseInfoExtractor):
                 list)
             if not slr_contents:
                 break
-            isr_contents = try_get(
-                slr_contents,
-                lambda x: x[0]['itemSectionRenderer']['contents'],
-                list)
-            if not isr_contents:
-                break
-            for content in isr_contents:
-                if not isinstance(content, dict):
+            for slr_content in slr_contents:
+                isr_contents = try_get(
+                    slr_content,
+                    lambda x: x['itemSectionRenderer']['contents'],
+                    list)
+                if not isr_contents:
                     continue
-                video = content.get('videoRenderer')
-                if not isinstance(video, dict):
-                    continue
-                video_id = video.get('videoId')
-                if not video_id:
-                    continue
-                title = try_get(video, lambda x: x['title']['runs'][0]['text'], compat_str)
-                description = try_get(video, lambda x: x['descriptionSnippet']['runs'][0]['text'], compat_str)
-                duration = parse_duration(try_get(video, lambda x: x['lengthText']['simpleText'], compat_str))
-                view_count_text = try_get(video, lambda x: x['viewCountText']['simpleText'], compat_str) or ''
-                view_count = int_or_none(self._search_regex(
-                    r'^(\d+)', re.sub(r'\s', '', view_count_text),
-                    'view count', default=None))
-                uploader = try_get(video, lambda x: x['ownerText']['runs'][0]['text'], compat_str)
-                total += 1
-                yield {
-                    '_type': 'url_transparent',
-                    'ie_key': YoutubeIE.ie_key(),
-                    'id': video_id,
-                    'url': video_id,
-                    'title': title,
-                    'description': description,
-                    'duration': duration,
-                    'view_count': view_count,
-                    'uploader': uploader,
-                }
-                if total == n:
-                    return
+                for content in isr_contents:
+                    if not isinstance(content, dict):
+                        continue
+                    video = content.get('videoRenderer')
+                    if not isinstance(video, dict):
+                        continue
+                    video_id = video.get('videoId')
+                    if not video_id:
+                        continue
+                    yield self._extract_video(video)
+                    total += 1
+                    if total == n:
+                        return
             token = try_get(
                 slr_contents,
-                lambda x: x[1]['continuationItemRenderer']['continuationEndpoint']['continuationCommand']['token'],
+                lambda x: x[-1]['continuationItemRenderer']['continuationEndpoint']['continuationCommand']['token'],
                 compat_str)
             if not token:
                 break

From a938f111ed2955dc5304d11c301bfa28df585fd3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adrian=20Heine=20n=C3=A9=20Lang?= <mail@adrianheine.de>
Date: Tue, 12 Jan 2021 21:12:44 +0100
Subject: [PATCH 108/860] [ADN] Fix extraction (#27732)

Closes #26963.
---
 youtube_dl/extractor/adn.py | 141 +++++++++++++++++++++---------------
 1 file changed, 84 insertions(+), 57 deletions(-)

diff --git a/youtube_dl/extractor/adn.py b/youtube_dl/extractor/adn.py
index c95ad2173..901832ac4 100644
--- a/youtube_dl/extractor/adn.py
+++ b/youtube_dl/extractor/adn.py
@@ -10,6 +10,7 @@ import random
 from .common import InfoExtractor
 from ..aes import aes_cbc_decrypt
 from ..compat import (
+    compat_HTTPError,
     compat_b64decode,
     compat_ord,
 )
@@ -18,10 +19,12 @@ from ..utils import (
     bytes_to_long,
     ExtractorError,
     float_or_none,
+    int_or_none,
     intlist_to_bytes,
     long_to_bytes,
     pkcs1pad,
     strip_or_none,
+    unified_strdate,
     urljoin,
 )
 
@@ -31,16 +34,18 @@ class ADNIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?animedigitalnetwork\.fr/video/[^/]+/(?P<id>\d+)'
     _TEST = {
         'url': 'http://animedigitalnetwork.fr/video/blue-exorcist-kyoto-saga/7778-episode-1-debut-des-hostilites',
-        'md5': 'e497370d847fd79d9d4c74be55575c7a',
+        'md5': '0319c99885ff5547565cacb4f3f9348d',
         'info_dict': {
             'id': '7778',
             'ext': 'mp4',
-            'title': 'Blue Exorcist - Kyôto Saga - Épisode 1',
+            'title': 'Blue Exorcist - Kyôto Saga - Episode 1',
             'description': 'md5:2f7b5aa76edbc1a7a92cedcda8a528d5',
         }
     }
+
     _BASE_URL = 'http://animedigitalnetwork.fr'
-    _RSA_KEY = (0xc35ae1e4356b65a73b551493da94b8cb443491c0aa092a357a5aee57ffc14dda85326f42d716e539a34542a0d3f363adf16c5ec222d713d5997194030ee2e4f0d1fb328c01a81cf6868c090d50de8e169c6b13d1675b9eeed1cbc51e1fffca9b38af07f37abd790924cd3bee59d0257cfda4fe5f3f0534877e21ce5821447d1b, 65537)
+    _API_BASE_URL = 'https://gw.api.animedigitalnetwork.fr'
+    _RSA_KEY = (0x9B42B08905199A5CCE2026274399CA560ECB209EE9878A708B1C0812E1BB8CB5D1FB7441861147C1A1F2F3A0476DD63A9CAC20D3E983613346850AA6CB38F16DC7D720FD7D86FC6E5B3D5BBC72E14CD0BF9E869F2CEA2CCAD648F1DCE38F1FF916CEFB2D339B64AA0264372344BC775E265E8A852F88144AB0BD9AA06C1A4ABB, 65537)
     _POS_ALIGN_MAP = {
         'start': 1,
         'end': 3,
@@ -119,59 +124,75 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        player_config = self._parse_json(self._search_regex(
-            r'playerConfig\s*=\s*({.+});', webpage,
-            'player config', default='{}'), video_id, fatal=False)
-        if not player_config:
-            config_url = urljoin(self._BASE_URL, self._search_regex(
-                r'(?:id="player"|class="[^"]*adn-player-container[^"]*")[^>]+data-url="([^"]+)"',
-                webpage, 'config url'))
-            player_config = self._download_json(
-                config_url, video_id,
-                'Downloading player config JSON metadata')['player']
+        config_url = self._API_BASE_URL + '/player/video/%s/configuration' % video_id
+        player_config = self._download_json(
+            config_url, video_id,
+            'Downloading player config JSON metadata')['player']['options']
 
-        video_info = {}
-        video_info_str = self._search_regex(
-            r'videoInfo\s*=\s*({.+});', webpage,
-            'video info', fatal=False)
-        if video_info_str:
-            video_info = self._parse_json(
-                video_info_str, video_id, fatal=False) or {}
+        user = player_config['user']
+        if not user.get('hasAccess'):
+            raise ExtractorError('This video is only available for paying users')
+            # self.raise_login_required() # FIXME: Login is not implemented
 
-        options = player_config.get('options') or {}
-        metas = options.get('metas') or {}
-        links = player_config.get('links') or {}
-        sub_path = player_config.get('subtitles')
-        error = None
-        if not links:
-            links_url = player_config.get('linksurl') or options['videoUrl']
-            token = options['token']
-            self._K = ''.join([random.choice('0123456789abcdef') for _ in range(16)])
-            message = bytes_to_intlist(json.dumps({
-                'k': self._K,
-                'e': 60,
-                't': token,
-            }))
+        token = self._download_json(
+            user.get('refreshTokenUrl') or (self._API_BASE_URL + '/player/refresh/token'),
+            video_id, 'Downloading access token', headers={'x-player-refresh-token': user['refreshToken']},
+            data=b'')['token']
+
+        links_url = player_config.get('videoUrl') or (self._API_BASE_URL + '/player/video/%s/link' % video_id)
+        self._K = ''.join([random.choice('0123456789abcdef') for _ in range(16)])
+        message = bytes_to_intlist(json.dumps({
+            'k': self._K,
+            't': token,
+        }))
+
+        # Sometimes authentication fails for no good reason, retry with
+        # a different random padding
+        links_data = None
+        for _ in range(3):
             padded_message = intlist_to_bytes(pkcs1pad(message, 128))
             n, e = self._RSA_KEY
             encrypted_message = long_to_bytes(pow(bytes_to_long(padded_message), e, n))
             authorization = base64.b64encode(encrypted_message).decode()
-            links_data = self._download_json(
-                urljoin(self._BASE_URL, links_url), video_id,
-                'Downloading links JSON metadata', headers={
-                    'Authorization': 'Bearer ' + authorization,
-                })
-            links = links_data.get('links') or {}
-            metas = metas or links_data.get('meta') or {}
-            sub_path = sub_path or links_data.get('subtitles') or \
-                'index.php?option=com_vodapi&task=subtitles.getJSON&format=json&id=' + video_id
-            sub_path += '&token=' + token
-            error = links_data.get('error')
-        title = metas.get('title') or video_info['title']
+
+            try:
+                links_data = self._download_json(
+                    urljoin(self._BASE_URL, links_url), video_id,
+                    'Downloading links JSON metadata', headers={
+                        'X-Player-Token': authorization
+                    },
+                    query={
+                        'freeWithAds': 'true',
+                        'adaptive': 'false',
+                        'withMetadata': 'true',
+                        'source': 'Web'
+                    }
+                )
+                break
+            except ExtractorError as e:
+                if not isinstance(e.cause, compat_HTTPError):
+                    raise e
+
+                if e.cause.code == 401:
+                    # This usually goes away with a different random pkcs1pad, so retry
+                    continue
+
+                error = self._parse_json(e.cause.read(), video_id)
+                message = error.get('message')
+                if e.cause.code == 403 and error.get('code') == 'player-bad-geolocation-country':
+                    self.raise_geo_restricted(msg=message)
+                else:
+                    raise ExtractorError(message)
+        else:
+            raise ExtractorError('Giving up retrying')
+
+        links = links_data.get('links') or {}
+        metas = links_data.get('metadata') or {}
+        sub_path = (links.get('subtitles') or {}).get('all')
+        video_info = links_data.get('video') or {}
 
         formats = []
-        for format_id, qualities in links.items():
+        for format_id, qualities in (links.get('streaming') or {}).items():
             if not isinstance(qualities, dict):
                 continue
             for quality, load_balancer_url in qualities.items():
@@ -189,19 +210,25 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
                     for f in m3u8_formats:
                         f['language'] = 'fr'
                 formats.extend(m3u8_formats)
-        if not error:
-            error = options.get('error')
-        if not formats and error:
-            raise ExtractorError('%s said: %s' % (self.IE_NAME, error), expected=True)
         self._sort_formats(formats)
 
+        video = (self._download_json(self._API_BASE_URL + '/video/%s' % video_id, video_id,
+                                     'Downloading additional video metadata', fatal=False) or {}).get('video')
+        show = video.get('show') or {}
+
         return {
             'id': video_id,
-            'title': title,
-            'description': strip_or_none(metas.get('summary') or video_info.get('resume')),
+            'title': metas.get('title') or video_id,
+            'description': strip_or_none(metas.get('summary') or video.get('summary')),
             'thumbnail': video_info.get('image'),
             'formats': formats,
-            'subtitles': self.extract_subtitles(sub_path, video_id),
-            'episode': metas.get('subtitle') or video_info.get('videoTitle'),
-            'series': video_info.get('playlistTitle'),
+            'subtitles': sub_path and self.extract_subtitles(sub_path, video_id),
+            'episode': metas.get('subtitle') or video.get('name'),
+            'episode_number': int_or_none(video.get('shortNumber')),
+            'series': video_info.get('playlistTitle') or show.get('title'),
+            'season_number': int_or_none(video.get('season')),
+            'duration': int_or_none(video_info.get('duration') or video.get('duration')),
+            'release_date': unified_strdate(video.get('release_date')),
+            'average_rating': video.get('rating') or metas.get('rating'),
+            'comment_count': int_or_none(video.get('commentsCount')),
         }

From cf862771d72b028f8d5335256610c560245dc8c0 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 12 Jan 2021 21:15:13 +0100
Subject: [PATCH 109/860] [adn] improve info extraction

---
 youtube_dl/extractor/adn.py | 79 +++++++++++++++++++++----------------
 1 file changed, 44 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/adn.py b/youtube_dl/extractor/adn.py
index 901832ac4..d611ee237 100644
--- a/youtube_dl/extractor/adn.py
+++ b/youtube_dl/extractor/adn.py
@@ -24,8 +24,8 @@ from ..utils import (
     long_to_bytes,
     pkcs1pad,
     strip_or_none,
+    try_get,
     unified_strdate,
-    urljoin,
 )
 
 
@@ -40,11 +40,20 @@ class ADNIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Blue Exorcist - Kyôto Saga - Episode 1',
             'description': 'md5:2f7b5aa76edbc1a7a92cedcda8a528d5',
+            'series': 'Blue Exorcist - Kyôto Saga',
+            'duration': 1467,
+            'release_date': '20170106',
+            'comment_count': int,
+            'average_rating': float,
+            'season_number': 2,
+            'episode': 'Début des hostilités',
+            'episode_number': 1,
         }
     }
 
     _BASE_URL = 'http://animedigitalnetwork.fr'
-    _API_BASE_URL = 'https://gw.api.animedigitalnetwork.fr'
+    _API_BASE_URL = 'https://gw.api.animedigitalnetwork.fr/'
+    _PLAYER_BASE_URL = _API_BASE_URL + 'player/'
     _RSA_KEY = (0x9B42B08905199A5CCE2026274399CA560ECB209EE9878A708B1C0812E1BB8CB5D1FB7441861147C1A1F2F3A0476DD63A9CAC20D3E983613346850AA6CB38F16DC7D720FD7D86FC6E5B3D5BBC72E14CD0BF9E869F2CEA2CCAD648F1DCE38F1FF916CEFB2D339B64AA0264372344BC775E265E8A852F88144AB0BD9AA06C1A4ABB, 65537)
     _POS_ALIGN_MAP = {
         'start': 1,
@@ -59,26 +68,24 @@ class ADNIE(InfoExtractor):
     def _ass_subtitles_timecode(seconds):
         return '%01d:%02d:%02d.%02d' % (seconds / 3600, (seconds % 3600) / 60, seconds % 60, (seconds % 1) * 100)
 
-    def _get_subtitles(self, sub_path, video_id):
-        if not sub_path:
+    def _get_subtitles(self, sub_url, video_id):
+        if not sub_url:
             return None
 
         enc_subtitles = self._download_webpage(
-            urljoin(self._BASE_URL, sub_path),
-            video_id, 'Downloading subtitles location', fatal=False) or '{}'
+            sub_url, video_id, 'Downloading subtitles location', fatal=False) or '{}'
         subtitle_location = (self._parse_json(enc_subtitles, video_id, fatal=False) or {}).get('location')
         if subtitle_location:
             enc_subtitles = self._download_webpage(
-                urljoin(self._BASE_URL, subtitle_location),
-                video_id, 'Downloading subtitles data', fatal=False,
-                headers={'Origin': 'https://animedigitalnetwork.fr'})
+                subtitle_location, video_id, 'Downloading subtitles data',
+                fatal=False, headers={'Origin': 'https://animedigitalnetwork.fr'})
         if not enc_subtitles:
             return None
 
         # http://animedigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js
         dec_subtitles = intlist_to_bytes(aes_cbc_decrypt(
             bytes_to_intlist(compat_b64decode(enc_subtitles[24:])),
-            bytes_to_intlist(binascii.unhexlify(self._K + '4b8ef13ec1872730')),
+            bytes_to_intlist(binascii.unhexlify(self._K + 'ab9f52f5baae7c72')),
             bytes_to_intlist(compat_b64decode(enc_subtitles[:24]))
         ))
         subtitles_json = self._parse_json(
@@ -124,22 +131,25 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        config_url = self._API_BASE_URL + '/player/video/%s/configuration' % video_id
-        player_config = self._download_json(
-            config_url, video_id,
-            'Downloading player config JSON metadata')['player']['options']
+        video_base_url = self._PLAYER_BASE_URL + 'video/%s/' % video_id
+        player = self._download_json(
+            video_base_url + 'configuration', video_id,
+            'Downloading player config JSON metadata')['player']
+        options = player['options']
 
-        user = player_config['user']
+        user = options['user']
         if not user.get('hasAccess'):
-            raise ExtractorError('This video is only available for paying users')
+            raise ExtractorError(
+                'This video is only available for paying users', expected=True)
             # self.raise_login_required() # FIXME: Login is not implemented
 
         token = self._download_json(
-            user.get('refreshTokenUrl') or (self._API_BASE_URL + '/player/refresh/token'),
-            video_id, 'Downloading access token', headers={'x-player-refresh-token': user['refreshToken']},
-            data=b'')['token']
+            user.get('refreshTokenUrl') or (self._PLAYER_BASE_URL + 'refresh/token'),
+            video_id, 'Downloading access token', headers={
+                'x-player-refresh-token': user['refreshToken']
+            }, data=b'')['token']
 
-        links_url = player_config.get('videoUrl') or (self._API_BASE_URL + '/player/video/%s/link' % video_id)
+        links_url = try_get(options, lambda x: x['video']['url']) or (video_base_url + 'link')
         self._K = ''.join([random.choice('0123456789abcdef') for _ in range(16)])
         message = bytes_to_intlist(json.dumps({
             'k': self._K,
@@ -157,17 +167,14 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
 
             try:
                 links_data = self._download_json(
-                    urljoin(self._BASE_URL, links_url), video_id,
-                    'Downloading links JSON metadata', headers={
+                    links_url, video_id, 'Downloading links JSON metadata', headers={
                         'X-Player-Token': authorization
-                    },
-                    query={
+                    }, query={
                         'freeWithAds': 'true',
                         'adaptive': 'false',
                         'withMetadata': 'true',
                         'source': 'Web'
-                    }
-                )
+                    })
                 break
             except ExtractorError as e:
                 if not isinstance(e.cause, compat_HTTPError):
@@ -188,8 +195,9 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
 
         links = links_data.get('links') or {}
         metas = links_data.get('metadata') or {}
-        sub_path = (links.get('subtitles') or {}).get('all')
+        sub_url = (links.get('subtitles') or {}).get('all')
         video_info = links_data.get('video') or {}
+        title = metas['title']
 
         formats = []
         for format_id, qualities in (links.get('streaming') or {}).items():
@@ -212,23 +220,24 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
                 formats.extend(m3u8_formats)
         self._sort_formats(formats)
 
-        video = (self._download_json(self._API_BASE_URL + '/video/%s' % video_id, video_id,
-                                     'Downloading additional video metadata', fatal=False) or {}).get('video')
+        video = (self._download_json(
+            self._API_BASE_URL + 'video/%s' % video_id, video_id,
+            'Downloading additional video metadata', fatal=False) or {}).get('video') or {}
         show = video.get('show') or {}
 
         return {
             'id': video_id,
-            'title': metas.get('title') or video_id,
+            'title': title,
             'description': strip_or_none(metas.get('summary') or video.get('summary')),
-            'thumbnail': video_info.get('image'),
+            'thumbnail': video_info.get('image') or player.get('image'),
             'formats': formats,
-            'subtitles': sub_path and self.extract_subtitles(sub_path, video_id),
+            'subtitles': self.extract_subtitles(sub_url, video_id),
             'episode': metas.get('subtitle') or video.get('name'),
             'episode_number': int_or_none(video.get('shortNumber')),
-            'series': video_info.get('playlistTitle') or show.get('title'),
+            'series': show.get('title'),
             'season_number': int_or_none(video.get('season')),
             'duration': int_or_none(video_info.get('duration') or video.get('duration')),
-            'release_date': unified_strdate(video.get('release_date')),
-            'average_rating': video.get('rating') or metas.get('rating'),
+            'release_date': unified_strdate(video.get('releaseDate')),
+            'average_rating': float_or_none(video.get('rating') or metas.get('rating')),
             'comment_count': int_or_none(video.get('commentsCount')),
         }

From 2408e6d26a9a1bd899769903ac28a8deaaa778a3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 13 Jan 2021 09:17:43 +0100
Subject: [PATCH 110/860] [cspan] improve info extraction(closes #27791)

---
 youtube_dl/extractor/cspan.py | 27 ++++++++++++++++++++++++++-
 1 file changed, 26 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cspan.py b/youtube_dl/extractor/cspan.py
index 766942146..2e01aff48 100644
--- a/youtube_dl/extractor/cspan.py
+++ b/youtube_dl/extractor/cspan.py
@@ -8,11 +8,14 @@ from ..utils import (
     ExtractorError,
     extract_attributes,
     find_xpath_attr,
+    get_element_by_attribute,
     get_element_by_class,
     int_or_none,
     js_to_json,
     merge_dicts,
+    parse_iso8601,
     smuggle_url,
+    str_to_int,
     unescapeHTML,
 )
 from .senateisvp import SenateISVPIE
@@ -116,8 +119,30 @@ class CSpanIE(InfoExtractor):
                 jwsetup, video_id, require_title=False, m3u8_id='hls',
                 base_url=url)
             add_referer(info['formats'])
+            for subtitles in info['subtitles'].values():
+                for subtitle in subtitles:
+                    ext = determine_ext(subtitle['url'])
+                    if ext == 'php':
+                        ext = 'vtt'
+                    subtitle['ext'] = ext
             ld_info = self._search_json_ld(webpage, video_id, default={})
-            return merge_dicts(info, ld_info)
+            title = get_element_by_class('video-page-title', webpage) or \
+                self._og_search_title(webpage)
+            description = get_element_by_attribute('itemprop', 'description', webpage) or \
+                self._html_search_meta(['og:description', 'description'], webpage)
+            return merge_dicts(info, ld_info, {
+                'title': title,
+                'thumbnail': get_element_by_attribute('itemprop', 'thumbnailUrl', webpage),
+                'description': description,
+                'timestamp': parse_iso8601(get_element_by_attribute('itemprop', 'uploadDate', webpage)),
+                'location': get_element_by_attribute('itemprop', 'contentLocation', webpage),
+                'duration': int_or_none(self._search_regex(
+                    r'jwsetup\.seclength\s*=\s*(\d+);',
+                    webpage, 'duration', fatal=False)),
+                'view_count': str_to_int(self._search_regex(
+                    r"<span[^>]+class='views'[^>]*>([\d,]+)\s+Views</span>",
+                    webpage, 'views', fatal=False)),
+            })
 
         # Obsolete
         # We first look for clipid, because clipprog always appears before

From 7c2d18a13f7eaa4008923ad73ed8e938229db848 Mon Sep 17 00:00:00 2001
From: Aarni Koskela <akx@iki.fi>
Date: Wed, 13 Jan 2021 11:22:48 +0200
Subject: [PATCH 111/860] [Mixcloud] Harmonize ID generation from lists with
 full ID generation (#27787)

Mixcloud IDs are generated as `username_slug` when the full ID dict has been
downloaded.  When downloading a list (e.g. uploads, favorites, ...), the temporary
ID is just the `slug`.  This made e.g. archive file usage require the download
of stream metadata before the download can be rejected as already downloaded.

This commit attempts to get the uploader username during the GraphQL query, so the
temporary IDs are generated similarly.
---
 youtube_dl/extractor/mixcloud.py | 11 +++++++++--
 1 file changed, 9 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 9759560f1..37f16a791 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -251,8 +251,13 @@ class MixcloudPlaylistBaseIE(MixcloudBaseIE):
                 cloudcast_url = cloudcast.get('url')
                 if not cloudcast_url:
                     continue
+                video_id = cloudcast.get('slug')
+                if video_id:
+                    owner_username = try_get(cloudcast, lambda x: x['owner']['username'], compat_str)
+                    if owner_username:
+                        video_id = '%s_%s' % (owner_username, video_id)
                 entries.append(self.url_result(
-                    cloudcast_url, MixcloudIE.ie_key(), cloudcast.get('slug')))
+                    cloudcast_url, MixcloudIE.ie_key(), video_id))
 
             page_info = items['pageInfo']
             has_next_page = page_info['hasNextPage']
@@ -321,7 +326,8 @@ class MixcloudUserIE(MixcloudPlaylistBaseIE):
     _DESCRIPTION_KEY = 'biog'
     _ROOT_TYPE = 'user'
     _NODE_TEMPLATE = '''slug
-          url'''
+          url
+          owner { username }'''
 
     def _get_playlist_title(self, title, slug):
         return '%s (%s)' % (title, slug)
@@ -345,6 +351,7 @@ class MixcloudPlaylistIE(MixcloudPlaylistBaseIE):
     _NODE_TEMPLATE = '''cloudcast {
             slug
             url
+            owner { username }
           }'''
 
     def _get_cloudcast(self, node):

From d81a213cfb66f6e3175980d763318bd81e31be02 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2021 00:37:51 +0700
Subject: [PATCH 112/860] [YoutubeDL] Raise syntax error for format selection
 expressions with multiple + operators (closes #27803)

---
 test/test_YoutubeDL.py  | 1 +
 youtube_dl/YoutubeDL.py | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 62f916d11..4d62ba145 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -464,6 +464,7 @@ class TestFormatSelection(unittest.TestCase):
         assert_syntax_error('+bestaudio')
         assert_syntax_error('bestvideo+')
         assert_syntax_error('/')
+        assert_syntax_error('bestvideo+bestvideo+bestaudio')
 
     def test_format_filtering(self):
         formats = [
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index aaac149e9..3f1f03379 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1226,6 +1226,8 @@ class YoutubeDL(object):
                         group = _parse_format_selection(tokens, inside_group=True)
                         current_selector = FormatSelector(GROUP, group, [])
                     elif string == '+':
+                        if inside_merge:
+                            raise syntax_error('Unexpected "+"', start)
                         video_selector = current_selector
                         audio_selector = _parse_format_selection(tokens, inside_merge=True)
                         if not video_selector or not audio_selector:

From 1e8e5d5238b4863aa2005d1cd5981343d96bdbf3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 14 Jan 2021 02:54:38 +0700
Subject: [PATCH 113/860] [workflows/ci.yml] Run CI on PRs

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index bac3256e7..a9dc47a71 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,5 +1,5 @@
 name: CI
-on: [push]
+on: [push, pull_request]
 jobs:
   tests:
     name: Tests

From 8e5477d036319adee98a6a7f95fc0ba09bb7b28c Mon Sep 17 00:00:00 2001
From: Aaron Zeng <zeng.aaron.l@gmail.com>
Date: Wed, 13 Jan 2021 23:46:07 -0500
Subject: [PATCH 114/860] [YoutubeDL] Ignore failure to create existing
 directory (#27811)

---
 youtube_dl/YoutubeDL.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3f1f03379..0ed4bc6ba 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1779,6 +1779,8 @@ class YoutubeDL(object):
                     os.makedirs(dn)
                 return True
             except (OSError, IOError) as err:
+                if isinstance(err, OSError) and err.errno == errno.EEXIST:
+                    return True
                 self.report_error('unable to create directory ' + error_to_compat_str(err))
                 return False
 

From 879866a2304c3b0bbbb048feb4253431f0219aa3 Mon Sep 17 00:00:00 2001
From: "main()" <main@ehvag.eu.org>
Date: Thu, 14 Jan 2021 13:41:44 +0100
Subject: [PATCH 115/860] [twitch] Set OAuth token for GraphQL requests using
 auth-token cookie (#27790)

Co-authored-by: remitamine <remitamine@gmail.com>
---
 youtube_dl/extractor/twitch.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index c4fd2f686..a7867f4d3 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -138,13 +138,17 @@ class TwitchBaseIE(InfoExtractor):
         self._sort_formats(formats)
 
     def _download_base_gql(self, video_id, ops, note, fatal=True):
+        headers = {
+            'Content-Type': 'text/plain;charset=UTF-8',
+            'Client-ID': self._CLIENT_ID,
+        }
+        gql_auth = self._get_cookies('https://gql.twitch.tv').get('auth-token')
+        if gql_auth:
+            headers['Authorization'] = 'OAuth ' + gql_auth.value
         return self._download_json(
             'https://gql.twitch.tv/gql', video_id, note,
             data=json.dumps(ops).encode(),
-            headers={
-                'Content-Type': 'text/plain;charset=UTF-8',
-                'Client-ID': self._CLIENT_ID,
-            }, fatal=fatal)
+            headers=headers, fatal=fatal)
 
     def _download_gql(self, video_id, ops, note, fatal=True):
         for op in ops:

From ab9001dab50db90f6470fcaf2189bcd2cfc0c370 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Jan 2021 12:38:15 +0700
Subject: [PATCH 116/860] [twitter] Add support for unified cards (closes
 #27826)

---
 youtube_dl/extractor/twitter.py | 28 ++++++++++++++++++++++++++--
 1 file changed, 26 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index 1190d721e..ec99dfccd 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -373,6 +373,24 @@ class TwitterIE(TwitterBaseIE):
             'uploader_id': '1eVjYOLGkGrQL',
         },
         'add_ie': ['TwitterBroadcast'],
+    }, {
+        # unified card
+        'url': 'https://twitter.com/BrooklynNets/status/1349794411333394432?s=20',
+        'info_dict': {
+            'id': '1349794411333394432',
+            'ext': 'mp4',
+            'title': 'md5:d1c4941658e4caaa6cb579260d85dcba',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'description': 'md5:71ead15ec44cee55071547d6447c6a3e',
+            'uploader': 'Brooklyn Nets',
+            'uploader_id': 'BrooklynNets',
+            'duration': 324.484,
+            'timestamp': 1610651040,
+            'upload_date': '20210114',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         # Twitch Clip Embed
         'url': 'https://twitter.com/GunB1g/status/1163218564784017422',
@@ -433,8 +451,7 @@ class TwitterIE(TwitterBaseIE):
             'tags': tags,
         }
 
-        media = try_get(status, lambda x: x['extended_entities']['media'][0])
-        if media and media.get('type') != 'photo':
+        def extract_from_video_info(media):
             video_info = media.get('video_info') or {}
 
             formats = []
@@ -461,6 +478,10 @@ class TwitterIE(TwitterBaseIE):
                 'thumbnails': thumbnails,
                 'duration': float_or_none(video_info.get('duration_millis'), 1000),
             })
+
+        media = try_get(status, lambda x: x['extended_entities']['media'][0])
+        if media and media.get('type') != 'photo':
+            extract_from_video_info(media)
         else:
             card = status.get('card')
             if card:
@@ -493,6 +514,9 @@ class TwitterIE(TwitterBaseIE):
                         '_type': 'url',
                         'url': get_binding_value('card_url'),
                     })
+                elif card_name == 'unified_card':
+                    media_entities = self._parse_json(get_binding_value('unified_card'), twid)['media_entities']
+                    extract_from_video_info(next(iter(media_entities.values())))
                 # amplify, promo_video_website, promo_video_convo, appplayer, ...
                 else:
                     is_amplify = card_name == 'amplify'

From b484097b01e4f864c7a4369fc6fd071f756802af Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Jan 2021 14:43:52 +0700
Subject: [PATCH 117/860] [youporn] Fix extraction (closes #27822)

---
 youtube_dl/extractor/youporn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 7b9feafeb..a1f0cce2c 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -88,7 +88,7 @@ class YouPornIE(InfoExtractor):
         # Main source
         definitions = self._parse_json(
             self._search_regex(
-                r'mediaDefinition\s*=\s*(\[.+?\]);', webpage,
+                r'mediaDefinition\s*[=:]\s*(\[.+?\])\s*[;,]', webpage,
                 'media definitions', default='[]'),
             video_id, fatal=False)
         if definitions:

From aa860b80161152e7205232529e00f3fe636d000e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Jan 2021 14:54:23 +0700
Subject: [PATCH 118/860] [youporn] Improve height and tbr extraction (refs
 #23659, refs #20425)

---
 youtube_dl/extractor/youporn.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index a1f0cce2c..4ca75454e 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -60,6 +60,9 @@ class YouPornIE(InfoExtractor):
     }, {
         'url': 'http://www.youporn.com/watch/505835',
         'only_matching': True,
+    }, {
+        'url': 'https://www.youporn.com/watch/13922959/femdom-principal/',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -128,8 +131,9 @@ class YouPornIE(InfoExtractor):
             # Video URL's path looks like this:
             #  /201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
             #  /201012/17/505835/vl_240p_240k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
+            #  /videos/201703/11/109285532/1080P_4000K_109285532.mp4
             # We will benefit from it by extracting some metadata
-            mobj = re.search(r'(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+/', video_url)
+            mobj = re.search(r'(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+', video_url)
             if mobj:
                 height = int(mobj.group('height'))
                 bitrate = int(mobj.group('bitrate'))

From 7e92f9015efe97352e824bb593bd810721f4a604 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 15 Jan 2021 15:12:04 +0700
Subject: [PATCH 119/860] [youporn] Restrict fallback download URL (refs
 #27822)

---
 youtube_dl/extractor/youporn.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 4ca75454e..534270bac 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -103,7 +103,7 @@ class YouPornIE(InfoExtractor):
                     links.append(video_url)
 
         # Fallback #1, this also contains extra low quality 180p format
-        for _, link in re.findall(r'<a[^>]+href=(["\'])(http.+?)\1[^>]+title=["\']Download [Vv]ideo', webpage):
+        for _, link in re.findall(r'<a[^>]+href=(["\'])(http(?:(?!\1).)+\.mp4(?:(?!\1).)*)\1[^>]+title=["\']Download [Vv]ideo', webpage):
             links.append(link)
 
         # Fallback #2 (unavailable as at 22.06.2017)

From 9d50f862323ed3d7c1ccd014c16d5d25b05de925 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 15 Jan 2021 10:32:01 +0100
Subject: [PATCH 120/860] [twitter] Add tests for more cards

---
 youtube_dl/extractor/twitter.py | 20 +++++++++++++++++++-
 1 file changed, 19 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index ec99dfccd..ed495f297 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -407,6 +407,22 @@ class TwitterIE(TwitterBaseIE):
         # appplayer card
         'url': 'https://twitter.com/poco_dandy/status/1150646424461176832',
         'only_matching': True,
+    }, {
+        # video_direct_message card
+        'url': 'https://twitter.com/qarev001/status/1348948114569269251',
+        'only_matching': True,
+    }, {
+        # poll2choice_video card
+        'url': 'https://twitter.com/CAF_Online/status/1349365911120195585',
+        'only_matching': True,
+    }, {
+        # poll3choice_video card
+        'url': 'https://twitter.com/SamsungMobileSA/status/1348609186725289984',
+        'only_matching': True,
+    }, {
+        # poll4choice_video card
+        'url': 'https://twitter.com/SouthamptonFC/status/1347577658079641604',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -517,7 +533,9 @@ class TwitterIE(TwitterBaseIE):
                 elif card_name == 'unified_card':
                     media_entities = self._parse_json(get_binding_value('unified_card'), twid)['media_entities']
                     extract_from_video_info(next(iter(media_entities.values())))
-                # amplify, promo_video_website, promo_video_convo, appplayer, ...
+                # amplify, promo_video_website, promo_video_convo, appplayer,
+                # video_direct_message, poll2choice_video, poll3choice_video,
+                # poll4choice_video, ...
                 else:
                     is_amplify = card_name == 'amplify'
                     vmap_url = get_binding_value('amplify_url_vmap') if is_amplify else get_binding_value('player_stream_url')

From 9c9b45814582cc763d45cda2742f9574f254c6a0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jan 2021 01:46:41 +0700
Subject: [PATCH 121/860] [YoutubeDL] Protect from infinite recursion due to
 recursively nested playlists (closes #27833)

---
 youtube_dl/YoutubeDL.py | 236 ++++++++++++++++++++++------------------
 1 file changed, 129 insertions(+), 107 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 0ed4bc6ba..efd42fa63 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -338,6 +338,8 @@ class YoutubeDL(object):
     _pps = []
     _download_retcode = None
     _num_downloads = None
+    _playlist_level = 0
+    _playlist_urls = set()
     _screen_file = None
 
     def __init__(self, params=None, auto_init=True):
@@ -906,115 +908,23 @@ class YoutubeDL(object):
             return self.process_ie_result(
                 new_result, download=download, extra_info=extra_info)
         elif result_type in ('playlist', 'multi_video'):
-            # We process each entry in the playlist
-            playlist = ie_result.get('title') or ie_result.get('id')
-            self.to_screen('[download] Downloading playlist: %s' % playlist)
-
-            playlist_results = []
-
-            playliststart = self.params.get('playliststart', 1) - 1
-            playlistend = self.params.get('playlistend')
-            # For backwards compatibility, interpret -1 as whole list
-            if playlistend == -1:
-                playlistend = None
-
-            playlistitems_str = self.params.get('playlist_items')
-            playlistitems = None
-            if playlistitems_str is not None:
-                def iter_playlistitems(format):
-                    for string_segment in format.split(','):
-                        if '-' in string_segment:
-                            start, end = string_segment.split('-')
-                            for item in range(int(start), int(end) + 1):
-                                yield int(item)
-                        else:
-                            yield int(string_segment)
-                playlistitems = orderedSet(iter_playlistitems(playlistitems_str))
-
-            ie_entries = ie_result['entries']
-
-            def make_playlistitems_entries(list_ie_entries):
-                num_entries = len(list_ie_entries)
-                return [
-                    list_ie_entries[i - 1] for i in playlistitems
-                    if -num_entries <= i - 1 < num_entries]
-
-            def report_download(num_entries):
+            # Protect from infinite recursion due to recursively nested playlists
+            # (see https://github.com/ytdl-org/youtube-dl/issues/27833)
+            webpage_url = ie_result['webpage_url']
+            if webpage_url in self._playlist_urls:
                 self.to_screen(
-                    '[%s] playlist %s: Downloading %d videos' %
-                    (ie_result['extractor'], playlist, num_entries))
+                    '[download] Skipping already downloaded playlist: %s'
+                    % ie_result.get('title') or ie_result.get('id'))
+                return
 
-            if isinstance(ie_entries, list):
-                n_all_entries = len(ie_entries)
-                if playlistitems:
-                    entries = make_playlistitems_entries(ie_entries)
-                else:
-                    entries = ie_entries[playliststart:playlistend]
-                n_entries = len(entries)
-                self.to_screen(
-                    '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
-                    (ie_result['extractor'], playlist, n_all_entries, n_entries))
-            elif isinstance(ie_entries, PagedList):
-                if playlistitems:
-                    entries = []
-                    for item in playlistitems:
-                        entries.extend(ie_entries.getslice(
-                            item - 1, item
-                        ))
-                else:
-                    entries = ie_entries.getslice(
-                        playliststart, playlistend)
-                n_entries = len(entries)
-                report_download(n_entries)
-            else:  # iterable
-                if playlistitems:
-                    entries = make_playlistitems_entries(list(itertools.islice(
-                        ie_entries, 0, max(playlistitems))))
-                else:
-                    entries = list(itertools.islice(
-                        ie_entries, playliststart, playlistend))
-                n_entries = len(entries)
-                report_download(n_entries)
-
-            if self.params.get('playlistreverse', False):
-                entries = entries[::-1]
-
-            if self.params.get('playlistrandom', False):
-                random.shuffle(entries)
-
-            x_forwarded_for = ie_result.get('__x_forwarded_for_ip')
-
-            for i, entry in enumerate(entries, 1):
-                self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
-                # This __x_forwarded_for_ip thing is a bit ugly but requires
-                # minimal changes
-                if x_forwarded_for:
-                    entry['__x_forwarded_for_ip'] = x_forwarded_for
-                extra = {
-                    'n_entries': n_entries,
-                    'playlist': playlist,
-                    'playlist_id': ie_result.get('id'),
-                    'playlist_title': ie_result.get('title'),
-                    'playlist_uploader': ie_result.get('uploader'),
-                    'playlist_uploader_id': ie_result.get('uploader_id'),
-                    'playlist_index': playlistitems[i - 1] if playlistitems else i + playliststart,
-                    'extractor': ie_result['extractor'],
-                    'webpage_url': ie_result['webpage_url'],
-                    'webpage_url_basename': url_basename(ie_result['webpage_url']),
-                    'extractor_key': ie_result['extractor_key'],
-                }
-
-                reason = self._match_entry(entry, incomplete=True)
-                if reason is not None:
-                    self.to_screen('[download] ' + reason)
-                    continue
-
-                entry_result = self.__process_iterable_entry(entry, download, extra)
-                # TODO: skip failed (empty) entries?
-                playlist_results.append(entry_result)
-            ie_result['entries'] = playlist_results
-            self.to_screen('[download] Finished downloading playlist: %s' % playlist)
-            return ie_result
+            self._playlist_level += 1
+            self._playlist_urls.add(webpage_url)
+            try:
+                return self.__process_playlist(ie_result, download)
+            finally:
+                self._playlist_level -= 1
+                if not self._playlist_level:
+                    self._playlist_urls.clear()
         elif result_type == 'compat_list':
             self.report_warning(
                 'Extractor %s returned a compat_list result. '
@@ -1039,6 +949,118 @@ class YoutubeDL(object):
         else:
             raise Exception('Invalid result type: %s' % result_type)
 
+    def __process_playlist(self, ie_result, download):
+        # We process each entry in the playlist
+        playlist = ie_result.get('title') or ie_result.get('id')
+
+        self.to_screen('[download] Downloading playlist: %s' % playlist)
+
+        playlist_results = []
+
+        playliststart = self.params.get('playliststart', 1) - 1
+        playlistend = self.params.get('playlistend')
+        # For backwards compatibility, interpret -1 as whole list
+        if playlistend == -1:
+            playlistend = None
+
+        playlistitems_str = self.params.get('playlist_items')
+        playlistitems = None
+        if playlistitems_str is not None:
+            def iter_playlistitems(format):
+                for string_segment in format.split(','):
+                    if '-' in string_segment:
+                        start, end = string_segment.split('-')
+                        for item in range(int(start), int(end) + 1):
+                            yield int(item)
+                    else:
+                        yield int(string_segment)
+            playlistitems = orderedSet(iter_playlistitems(playlistitems_str))
+
+        ie_entries = ie_result['entries']
+
+        def make_playlistitems_entries(list_ie_entries):
+            num_entries = len(list_ie_entries)
+            return [
+                list_ie_entries[i - 1] for i in playlistitems
+                if -num_entries <= i - 1 < num_entries]
+
+        def report_download(num_entries):
+            self.to_screen(
+                '[%s] playlist %s: Downloading %d videos' %
+                (ie_result['extractor'], playlist, num_entries))
+
+        if isinstance(ie_entries, list):
+            n_all_entries = len(ie_entries)
+            if playlistitems:
+                entries = make_playlistitems_entries(ie_entries)
+            else:
+                entries = ie_entries[playliststart:playlistend]
+            n_entries = len(entries)
+            self.to_screen(
+                '[%s] playlist %s: Collected %d video ids (downloading %d of them)' %
+                (ie_result['extractor'], playlist, n_all_entries, n_entries))
+        elif isinstance(ie_entries, PagedList):
+            if playlistitems:
+                entries = []
+                for item in playlistitems:
+                    entries.extend(ie_entries.getslice(
+                        item - 1, item
+                    ))
+            else:
+                entries = ie_entries.getslice(
+                    playliststart, playlistend)
+            n_entries = len(entries)
+            report_download(n_entries)
+        else:  # iterable
+            if playlistitems:
+                entries = make_playlistitems_entries(list(itertools.islice(
+                    ie_entries, 0, max(playlistitems))))
+            else:
+                entries = list(itertools.islice(
+                    ie_entries, playliststart, playlistend))
+            n_entries = len(entries)
+            report_download(n_entries)
+
+        if self.params.get('playlistreverse', False):
+            entries = entries[::-1]
+
+        if self.params.get('playlistrandom', False):
+            random.shuffle(entries)
+
+        x_forwarded_for = ie_result.get('__x_forwarded_for_ip')
+
+        for i, entry in enumerate(entries, 1):
+            self.to_screen('[download] Downloading video %s of %s' % (i, n_entries))
+            # This __x_forwarded_for_ip thing is a bit ugly but requires
+            # minimal changes
+            if x_forwarded_for:
+                entry['__x_forwarded_for_ip'] = x_forwarded_for
+            extra = {
+                'n_entries': n_entries,
+                'playlist': playlist,
+                'playlist_id': ie_result.get('id'),
+                'playlist_title': ie_result.get('title'),
+                'playlist_uploader': ie_result.get('uploader'),
+                'playlist_uploader_id': ie_result.get('uploader_id'),
+                'playlist_index': playlistitems[i - 1] if playlistitems else i + playliststart,
+                'extractor': ie_result['extractor'],
+                'webpage_url': ie_result['webpage_url'],
+                'webpage_url_basename': url_basename(ie_result['webpage_url']),
+                'extractor_key': ie_result['extractor_key'],
+            }
+
+            reason = self._match_entry(entry, incomplete=True)
+            if reason is not None:
+                self.to_screen('[download] ' + reason)
+                continue
+
+            entry_result = self.__process_iterable_entry(entry, download, extra)
+            # TODO: skip failed (empty) entries?
+            playlist_results.append(entry_result)
+        ie_result['entries'] = playlist_results
+        self.to_screen('[download] Finished downloading playlist: %s' % playlist)
+        return ie_result
+
     @__handle_extraction_exceptions
     def __process_iterable_entry(self, entry, download, extra_info):
         return self.process_ie_result(

From 0cd4c402f0530d357bb11b660e14a303290f6caf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jan 2021 09:22:33 +0700
Subject: [PATCH 122/860] [animeondemand] Add support for lazy playlist
 extraction (closes #27829)

---
 youtube_dl/extractor/animeondemand.py | 26 ++++++++++++++++----------
 1 file changed, 16 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/animeondemand.py b/youtube_dl/extractor/animeondemand.py
index 00ce684d1..54e097d2f 100644
--- a/youtube_dl/extractor/animeondemand.py
+++ b/youtube_dl/extractor/animeondemand.py
@@ -116,8 +116,6 @@ class AnimeOnDemandIE(InfoExtractor):
             r'(?s)<div[^>]+itemprop="description"[^>]*>(.+?)</div>',
             webpage, 'anime description', default=None)
 
-        entries = []
-
         def extract_info(html, video_id, num=None):
             title, description = [None] * 2
             formats = []
@@ -233,7 +231,7 @@ class AnimeOnDemandIE(InfoExtractor):
                 self._sort_formats(info['formats'])
                 f = common_info.copy()
                 f.update(info)
-                entries.append(f)
+                yield f
 
             # Extract teaser/trailer only when full episode is not available
             if not info['formats']:
@@ -247,7 +245,7 @@ class AnimeOnDemandIE(InfoExtractor):
                         'title': m.group('title'),
                         'url': urljoin(url, m.group('href')),
                     })
-                    entries.append(f)
+                    yield f
 
         def extract_episodes(html):
             for num, episode_html in enumerate(re.findall(
@@ -275,7 +273,8 @@ class AnimeOnDemandIE(InfoExtractor):
                     'episode_number': episode_number,
                 }
 
-                extract_entries(episode_html, video_id, common_info)
+                for e in extract_entries(episode_html, video_id, common_info):
+                    yield e
 
         def extract_film(html, video_id):
             common_info = {
@@ -283,11 +282,18 @@ class AnimeOnDemandIE(InfoExtractor):
                 'title': anime_title,
                 'description': anime_description,
             }
-            extract_entries(html, video_id, common_info)
+            for e in extract_entries(html, video_id, common_info):
+                yield e
 
-        extract_episodes(webpage)
+        def entries():
+            has_episodes = False
+            for e in extract_episodes(webpage):
+                has_episodes = True
+                yield e
 
-        if not entries:
-            extract_film(webpage, anime_id)
+            if not has_episodes:
+                for e in extract_film(webpage, anime_id):
+                    yield e
 
-        return self.playlist_result(entries, anime_id, anime_title, anime_description)
+        return self.playlist_result(
+            entries(), anime_id, anime_title, anime_description)

From f1487d4fca40fd37d735753e24a7bae53a1b1513 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jan 2021 09:42:11 +0700
Subject: [PATCH 123/860] [mixcloud:playlist:base] Fix video id extraction in
 flat playlist mode (refs #27787)

---
 youtube_dl/extractor/mixcloud.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 37f16a791..69319857d 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -251,11 +251,9 @@ class MixcloudPlaylistBaseIE(MixcloudBaseIE):
                 cloudcast_url = cloudcast.get('url')
                 if not cloudcast_url:
                     continue
-                video_id = cloudcast.get('slug')
-                if video_id:
-                    owner_username = try_get(cloudcast, lambda x: x['owner']['username'], compat_str)
-                    if owner_username:
-                        video_id = '%s_%s' % (owner_username, video_id)
+                slug = try_get(cloudcast, lambda x: x['slug'], compat_str)
+                owner_username = try_get(cloudcast, lambda x: x['owner']['username'], compat_str)
+                video_id = '%s_%s' % (owner_username, slug) if slug and owner_username else None
                 entries.append(self.url_result(
                     cloudcast_url, MixcloudIE.ie_key(), video_id))
 

From 8673f4344c40bf771af5344113b184f5cef08030 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jan 2021 09:48:00 +0700
Subject: [PATCH 124/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 3629c4fb8..ab057fae1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,34 @@
+version <unreleased>
+
+Core
+* [YoutubeDL] Protect from infinite recursion due to recursively nested
+  playlists (#27833)
+* [YoutubeDL] Ignore failure to create existing directory (#27811)
+* [YoutubeDL] Raise syntax error for format selection expressions with multiple
+  + operators (#27803)
+
+Extractors
++ [animeondemand] Add support for lazy playlist extraction (#27829)
+* [youporn] Restrict fallback download URL (#27822)
+* [youporn] Improve height and tbr extraction (#20425, #23659)
+* [youporn] Fix extraction (#27822)
++ [twitter] Add support for unified cards (#27826)
++ [twitch] Add Authorization header with OAuth token for GraphQL requests
+  (#27790)
+* [mixcloud:playlist:base] Extract video id in flat playlist mode (#27787)
+* [cspan] Improve info extraction (#27791)
+* [adn] Improve info extraction
+* [adn] Fix extraction (#26963, #27732)
+* [youtube:search] Extract from all sections (#27604)
+* [youtube:search] fix viewcount and try to extract all video sections (#27604)
+* [twitch] Improve login error extraction
+* [twitch] Fix authentication (#27743)
+* [3qsdn] Improve extraction (#21058)
+* [peertube] Extract formats from streamingPlaylists (#26002, #27586, #27728)
+* [khanacademy] Fix extraction (#2887, #26803)
+* [spike] Update Paramount Network feed URL (#27715)
+
+
 version 2021.01.08
 
 Core

From ef50cb3fda7c5455b036df648319c2829d899d7d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 16 Jan 2021 09:51:30 +0700
Subject: [PATCH 125/860] release 2021.01.16

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 docs/supportedsites.md                           | 3 ++-
 youtube_dl/version.py                            | 2 +-
 8 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 768d45fc1..aedcfa6b3 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.08. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.01.08**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.16**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.01.08
+ [debug] youtube-dl version 2021.01.16
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index 2bd90da57..5c0dfea4e 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.08. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.01.08**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.16**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 272895b47..772147a75 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.08. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.01.08**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.16**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index 608fcfba4..2fcaa3a23 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.08. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.01.08**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.16**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.01.08
+ [debug] youtube-dl version 2021.01.16
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index d085ab1ef..f1adfce8f 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.08. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.01.08**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.16**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index ab057fae1..27f01c438 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.01.16
 
 Core
 * [YoutubeDL] Protect from infinite recursion due to recursively nested
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 3a49043fa..aa8026a32 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -418,7 +418,8 @@
  - **Katsomo**
  - **KeezMovies**
  - **Ketnet**
- - **KhanAcademy**
+ - **khanacademy**
+ - **khanacademy:unit**
  - **KickStarter**
  - **KinjaEmbed**
  - **KinoPoisk**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0d9659b2b..ac7242abb 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.01.08'
+__version__ = '2021.01.16'

From 55a3ca16d3724376385801873c918e450a863f4a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 16 Jan 2021 18:12:05 +0100
Subject: [PATCH 126/860] [spotify] Add new extractor for Spotify
 Podcasts(closes #27443)

---
 youtube_dl/extractor/extractors.py |   4 +
 youtube_dl/extractor/spotify.py    | 156 +++++++++++++++++++++++++++++
 2 files changed, 160 insertions(+)
 create mode 100644 youtube_dl/extractor/spotify.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 57d4d319c..c50e1419f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1116,6 +1116,10 @@ from .stitcher import (
 from .sport5 import Sport5IE
 from .sportbox import SportBoxIE
 from .sportdeutschland import SportDeutschlandIE
+from .spotify import (
+    SpotifyIE,
+    SpotifyShowIE,
+)
 from .spreaker import (
     SpreakerIE,
     SpreakerPageIE,
diff --git a/youtube_dl/extractor/spotify.py b/youtube_dl/extractor/spotify.py
new file mode 100644
index 000000000..826f98cff
--- /dev/null
+++ b/youtube_dl/extractor/spotify.py
@@ -0,0 +1,156 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_podcast_url,
+    float_or_none,
+    int_or_none,
+    strip_or_none,
+    try_get,
+    unified_strdate,
+)
+
+
+class SpotifyBaseIE(InfoExtractor):
+    _ACCESS_TOKEN = None
+    _OPERATION_HASHES = {
+        'Episode': '8276d4423d709ae9b68ec1b74cc047ba0f7479059a37820be730f125189ac2bf',
+        'MinimalShow': '13ee079672fad3f858ea45a55eb109553b4fb0969ed793185b2e34cbb6ee7cc0',
+        'ShowEpisodes': 'e0e5ce27bd7748d2c59b4d44ba245a8992a05be75d6fabc3b20753fc8857444d',
+    }
+    _VALID_URL_TEMPL = r'https?://open\.spotify\.com/%s/(?P<id>[^/?&#]+)'
+
+    def _real_initialize(self):
+        self._ACCESS_TOKEN = self._download_json(
+            'https://open.spotify.com/get_access_token', None)['accessToken']
+
+    def _call_api(self, operation, video_id, variables):
+        return self._download_json(
+            'https://api-partner.spotify.com/pathfinder/v1/query', video_id, query={
+                'operationName': 'query' + operation,
+                'variables': json.dumps(variables),
+                'extensions': json.dumps({
+                    'persistedQuery': {
+                        'sha256Hash': self._OPERATION_HASHES[operation],
+                    },
+                })
+            }, headers={'authorization': 'Bearer ' + self._ACCESS_TOKEN})['data']
+
+    def _extract_episode(self, episode, series):
+        episode_id = episode['id']
+        title = episode['name'].strip()
+
+        formats = []
+        audio_preview = episode.get('audioPreview') or {}
+        audio_preview_url = audio_preview.get('url')
+        if audio_preview_url:
+            f = {
+                'url': audio_preview_url.replace('://p.scdn.co/mp3-preview/', '://anon-podcast.scdn.co/'),
+                'vcodec': 'none',
+            }
+            audio_preview_format = audio_preview.get('format')
+            if audio_preview_format:
+                f['format_id'] = audio_preview_format
+                mobj = re.match(r'([0-9A-Z]{3})_(?:[A-Z]+_)?(\d+)', audio_preview_format)
+                if mobj:
+                    f.update({
+                        'abr': int(mobj.group(2)),
+                        'ext': mobj.group(1).lower(),
+                    })
+            formats.append(f)
+
+        for item in (try_get(episode, lambda x: x['audio']['items']) or []):
+            item_url = item.get('url')
+            if not (item_url and item.get('externallyHosted')):
+                continue
+            formats.append({
+                'url': clean_podcast_url(item_url),
+                'vcodec': 'none',
+            })
+
+        thumbnails = []
+        for source in (try_get(episode, lambda x: x['coverArt']['sources']) or []):
+            source_url = source.get('url')
+            if not source_url:
+                continue
+            thumbnails.append({
+                'url': source_url,
+                'width': int_or_none(source.get('width')),
+                'height': int_or_none(source.get('height')),
+            })
+
+        return {
+            'id': episode_id,
+            'title': title,
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'description': strip_or_none(episode.get('description')),
+            'duration': float_or_none(try_get(
+                episode, lambda x: x['duration']['totalMilliseconds']), 1000),
+            'release_date': unified_strdate(try_get(
+                episode, lambda x: x['releaseDate']['isoString'])),
+            'series': series,
+        }
+
+
+class SpotifyIE(SpotifyBaseIE):
+    IE_NAME = 'spotify'
+    _VALID_URL = SpotifyBaseIE._VALID_URL_TEMPL % 'episode'
+    _TEST = {
+        'url': 'https://open.spotify.com/episode/4Z7GAJ50bgctf6uclHlWKo',
+        'md5': '74010a1e3fa4d9e1ab3aa7ad14e42d3b',
+        'info_dict': {
+            'id': '4Z7GAJ50bgctf6uclHlWKo',
+            'ext': 'mp3',
+            'title': 'From the archive: Why time management is ruining our lives',
+            'description': 'md5:b120d9c4ff4135b42aa9b6d9cde86935',
+            'duration': 2083.605,
+            'release_date': '20201217',
+            'series': "The Guardian's Audio Long Reads",
+        }
+    }
+
+    def _real_extract(self, url):
+        episode_id = self._match_id(url)
+        episode = self._call_api('Episode', episode_id, {
+            'uri': 'spotify:episode:' + episode_id
+        })['episode']
+        return self._extract_episode(
+            episode, try_get(episode, lambda x: x['podcast']['name']))
+
+
+class SpotifyShowIE(SpotifyBaseIE):
+    IE_NAME = 'spotify:show'
+    _VALID_URL = SpotifyBaseIE._VALID_URL_TEMPL % 'show'
+    _TEST = {
+        'url': 'https://open.spotify.com/show/4PM9Ke6l66IRNpottHKV9M',
+        'info_dict': {
+            'id': '4PM9Ke6l66IRNpottHKV9M',
+            'title': 'The Story from the Guardian',
+            'description': 'The Story podcast is dedicated to our finest audio documentaries, investigations and long form stories',
+        },
+        'playlist_mincount': 36,
+    }
+
+    def _real_extract(self, url):
+        show_id = self._match_id(url)
+        podcast = self._call_api('ShowEpisodes', show_id, {
+            'limit': 1000000000,
+            'offset': 0,
+            'uri': 'spotify:show:' + show_id,
+        })['podcast']
+        podcast_name = podcast.get('name')
+
+        entries = []
+        for item in (try_get(podcast, lambda x: x['episodes']['items']) or []):
+            episode = item.get('episode')
+            if not episode:
+                continue
+            entries.append(self._extract_episode(episode, podcast_name))
+
+        return self.playlist_result(
+            entries, show_id, podcast_name, podcast.get('description'))

From 360a5e0f60f273a0fd2fd664fed1439430cf35e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adrian=20Heine=20n=C3=A9=20Lang?= <mail@adrianheine.de>
Date: Sat, 16 Jan 2021 21:40:08 +0100
Subject: [PATCH 127/860] [aenetworks] Fix test (#27847)

---
 youtube_dl/extractor/aenetworks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 8e4963131..a5d88ebbe 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -256,7 +256,7 @@ class AENetworksShowIE(AENetworksListBaseIE):
             'title': 'Ancient Aliens',
             'description': 'md5:3f6d74daf2672ff3ae29ed732e37ea7f',
         },
-        'playlist_mincount': 168,
+        'playlist_mincount': 150,
     }]
     _RESOURCE = 'series'
     _ITEMS_KEY = 'episodes'

From d4564afc7074a0c12e62649a50a05a2cdaef4650 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adrian=20Heine=20n=C3=A9=20Lang?= <mail@adrianheine.de>
Date: Sat, 16 Jan 2021 23:41:52 +0100
Subject: [PATCH 128/860] [ard] Fix title and description extraction and update
 tests (#27761)

---
 youtube_dl/extractor/ard.py | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 5b7b2dd6d..6bf5b3f13 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -187,13 +187,13 @@ class ARDMediathekIE(ARDMediathekBaseIE):
             if doc.tag == 'rss':
                 return GenericIE()._extract_rss(url, video_id, doc)
 
-        title = self._html_search_regex(
+        title = self._og_search_title(webpage, default=None) or self._html_search_regex(
             [r'<h1(?:\s+class="boxTopHeadline")?>(.*?)</h1>',
              r'<meta name="dcterms\.title" content="(.*?)"/>',
              r'<h4 class="headline">(.*?)</h4>',
              r'<title[^>]*>(.*?)</title>'],
             webpage, 'title')
-        description = self._html_search_meta(
+        description = self._og_search_description(webpage, default=None) or self._html_search_meta(
             'dcterms.abstract', webpage, 'description', default=None)
         if description is None:
             description = self._html_search_meta(
@@ -249,18 +249,18 @@ class ARDMediathekIE(ARDMediathekBaseIE):
 
 
 class ARDIE(InfoExtractor):
-    _VALID_URL = r'(?P<mainurl>https?://(www\.)?daserste\.de/[^?#]+/videos(?:extern)?/(?P<display_id>[^/?#]+)-(?P<id>[0-9]+))\.html'
+    _VALID_URL = r'(?P<mainurl>https?://(?:www\.)?daserste\.de/[^?#]+/videos(?:extern)?/(?P<display_id>[^/?#]+)-(?:video-?)?(?P<id>[0-9]+))\.html'
     _TESTS = [{
-        # available till 14.02.2019
-        'url': 'http://www.daserste.de/information/talk/maischberger/videos/das-groko-drama-zerlegen-sich-die-volksparteien-video-102.html',
-        'md5': '8e4ec85f31be7c7fc08a26cdbc5a1f49',
+        # available till 7.01.2022
+        'url': 'https://www.daserste.de/information/talk/maischberger/videos/maischberger-die-woche-video100.html',
+        'md5': '867d8aa39eeaf6d76407c5ad1bb0d4c1',
         'info_dict': {
-            'display_id': 'das-groko-drama-zerlegen-sich-die-volksparteien-video',
-            'id': '102',
+            'display_id': 'maischberger-die-woche',
+            'id': '100',
             'ext': 'mp4',
-            'duration': 4435.0,
-            'title': 'Das GroKo-Drama: Zerlegen sich die Volksparteien?',
-            'upload_date': '20180214',
+            'duration': 3687.0,
+            'title': 'maischberger. die woche vom 7. Januar 2021',
+            'upload_date': '20210107',
             'thumbnail': r're:^https?://.*\.jpg$',
         },
     }, {
@@ -315,17 +315,17 @@ class ARDIE(InfoExtractor):
 class ARDBetaMediathekIE(ARDMediathekBaseIE):
     _VALID_URL = r'https://(?:(?:beta|www)\.)?ardmediathek\.de/(?P<client>[^/]+)/(?:player|live|video)/(?P<display_id>(?:[^/]+/)*)(?P<video_id>[a-zA-Z0-9]+)'
     _TESTS = [{
-        'url': 'https://ardmediathek.de/ard/video/die-robuste-roswita/Y3JpZDovL2Rhc2Vyc3RlLmRlL3RhdG9ydC9mYmM4NGM1NC0xNzU4LTRmZGYtYWFhZS0wYzcyZTIxNGEyMDE',
-        'md5': 'dfdc87d2e7e09d073d5a80770a9ce88f',
+        'url': 'https://www.ardmediathek.de/mdr/video/die-robuste-roswita/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy84MWMxN2MzZC0wMjkxLTRmMzUtODk4ZS0wYzhlOWQxODE2NGI/',
+        'md5': 'a1dc75a39c61601b980648f7c9f9f71d',
         'info_dict': {
             'display_id': 'die-robuste-roswita',
-            'id': '70153354',
+            'id': '78566716',
             'title': 'Die robuste Roswita',
-            'description': r're:^Der Mord.*trüber ist als die Ilm.',
+            'description': r're:^Der Mord.*totgeglaubte Ehefrau Roswita',
             'duration': 5316,
-            'thumbnail': 'https://img.ardmediathek.de/standard/00/70/15/33/90/-1852531467/16x9/960?mandant=ard',
-            'timestamp': 1577047500,
-            'upload_date': '20191222',
+            'thumbnail': 'https://img.ardmediathek.de/standard/00/78/56/67/84/575672121/16x9/960?mandant=ard',
+            'timestamp': 1596658200,
+            'upload_date': '20200805',
             'ext': 'mp4',
         },
     }, {

From de026a6acd3cbc2e62d0988b21c97e71e3730cb0 Mon Sep 17 00:00:00 2001
From: Tatsh <Tatsh@users.noreply.github.com>
Date: Sun, 17 Jan 2021 08:05:39 -0500
Subject: [PATCH 129/860] [Minds] Add new extractor (#17934)

---
 youtube_dl/extractor/extractors.py |   5 +
 youtube_dl/extractor/minds.py      | 164 +++++++++++++++++++++++++++++
 2 files changed, 169 insertions(+)
 create mode 100644 youtube_dl/extractor/minds.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index c50e1419f..90012fc4f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -651,6 +651,11 @@ from .microsoftvirtualacademy import (
     MicrosoftVirtualAcademyIE,
     MicrosoftVirtualAcademyCourseIE,
 )
+from .minds import (
+    MindsIE,
+    MindsActivityIE,
+    MindsChannelIE,
+)
 from .ministrygrid import MinistryGridIE
 from .minoto import MinotoIE
 from .miomio import MioMioIE
diff --git a/youtube_dl/extractor/minds.py b/youtube_dl/extractor/minds.py
new file mode 100644
index 000000000..4523d0938
--- /dev/null
+++ b/youtube_dl/extractor/minds.py
@@ -0,0 +1,164 @@
+# coding: utf-8
+from __future__ import unicode_literals
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (int_or_none, sanitized_Request, str_or_none,
+                     unified_strdate)
+
+
+class MindsIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?minds\.com/media/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://www.minds.com/media/100000000000086822',
+        'md5': '215a658184a419764852239d4970b045',
+        'info_dict': {
+            'id': '100000000000086822',
+            'ext': 'mp4',
+            'title': 'Minds intro sequence',
+            'thumbnail': 'https://cdn-cinemr.minds.com/cinemr_com/334128440657580032/thumbnail-00001.png',
+            'uploader_id': '100000000000000341',
+            'description': '<?xml encoding="utf-8" ?>',
+            'upload_date': '20130524',
+            'timestamp': 1369404826,
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video_api_url = 'https://www.minds.com/api/v1/media/%s' % video_id
+        token = self._get_cookies(url).get('XSRF-TOKEN')
+        headers = {
+            'authority': 'www.minds.com',
+            'referer': url,
+            'x-xsrf-token': token.value if token else '',
+        }
+        data = self._download_json(video_api_url, video_id, headers=headers,
+                                   query={'children': 'false'})
+        formats = []
+        owner = data.get('ownerObj', {})
+
+        transcodes = data.get('transcodes', {})
+        # These keys are the width so keep the highest width last
+        keys = sorted(transcodes.keys())
+
+        for format_id in keys:
+            is_numeric = re.match('^[0-9]+\.mp4', format_id)
+            video_url = transcodes[format_id]
+            info = {
+                'url': video_url,
+                'format_id': format_id,
+                'http_headers': headers,
+            }
+            if is_numeric:
+                info['width'] = int(format_id.split('.')[0])
+            formats.append(info)
+
+        uploader_id = str_or_none(owner.get('guid') or
+                                  data.get('owner_guid') or
+                                  owner.get('legacy_guid') or
+                                  owner.get('owner_guid'))
+        description = str_or_none(data.get('description'))
+        if description:
+            description = description.strip()
+        uploader_url = age_limit = thumbnail = None
+
+        if owner.get('username'):
+            uploader_url = 'https://www.minds.com/%s' % owner.get('username')
+        if data.get('mature') is True:
+            age_limit = 18
+
+        thumbnail_api_url = data.get('thumbnail_src')
+        if thumbnail_api_url:
+            req = sanitized_Request(thumbnail_api_url)
+            req.get_method = lambda: 'HEAD'
+            res = self._request_webpage(req, video_id)
+            if res.headers.get('content-type', '').startswith('image/'):
+                thumbnail = getattr(res, 'url', None)
+        tags = data.get('tags', '').strip()
+        if isinstance(tags, compat_str) and tags:
+            tags = [x.strip() for x in tags.split(',')]
+        else:
+            tags = None
+        category = data.get('category')
+        if isinstance(category, compat_str) and category:
+            category = [category]
+        else:
+            category = None
+
+        return {
+            'id': video_id,
+            'title': data['title'],
+            'formats': formats,
+            'description': description,
+            'license': str_or_none(data.get('license')),
+            'creator': str_or_none(owner.get('name') or owner.get('username')),
+            'release_date': unified_strdate(data.get('time_created')),
+            'timestamp': int_or_none(data.get('time_created')),
+            'uploader_id': uploader_id,
+            'uploader_url': uploader_url,
+            'view_count': int_or_none(data.get('play:count')),
+            'like_count': int_or_none(data.get('thumbs:up:count')),
+            'dislike_count': int_or_none(data.get('thumbs:down:count')),
+            'average_rating': int_or_none(data.get('rating')),
+            'age_limit': age_limit,
+            'categories': [str_or_none(data.get('category'))],
+            'tags': tags,
+            # As of 20181020 the API is returning `false` for this value both
+            # at top level and within the entity.comments:count path. The only
+            # other way to get this is to fetch all comments and count.
+            'comment_count': int_or_none(data.get('comments:count')),
+            'thumbnail': thumbnail,
+        }
+
+
+class MindsActivityIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?minds\.com/newsfeed/(?P<id>[0-9]+)'
+
+    def _real_extract(self, url):
+        guid = self._match_id(url)
+        api_url = 'https://www.minds.com/api/v1/newsfeed/single/%s' % guid
+        token = self._get_cookies(url).get('XSRF-TOKEN')
+        headers = {
+            'authority': 'www.minds.com',
+            'referer': url,
+            'x-xsrf-token': token.value if token else '',
+        }
+        data = self._download_json(api_url, guid, headers=headers)
+        return self.url_result('https://www.minds.com/media/%s' % data['activity']['entity_guid'])
+
+
+class MindsChannelIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?minds\.com/(?!newsfeed|media|api)(?P<id>[^/]+)'
+
+    def _real_extract(self, url):
+        channel_name = self._match_id(url)
+        api_url = 'https://www.minds.com/api/v1/channel/%s' % channel_name
+        token = self._get_cookies(url).get('XSRF-TOKEN')
+        headers = {
+            'authority': 'www.minds.com',
+            'referer': url,
+            'x-xsrf-token': token.value if token else '',
+        }
+        data = self._download_json(api_url, channel_name, headers=headers)
+        channel = data.get('channel', {})
+        params = {'limit': 12, 'offset': ''}
+        api_url = 'https://www.minds.com/api/v1/newsfeed/personal/%s' % channel['guid']
+        entries = []
+        while True:
+            data = self._download_json(api_url, channel['guid'],
+                                       headers=headers, query=params)
+            activity = data.get('activity', [])
+            if len(activity) == 0 or not data.get('load-next'):
+                break
+            for info in activity:
+                if info.get('custom_type') != 'video':
+                    continue
+                entries.append(self.url_result('https://www.minds.com/media/%s' % info['entity_guid']))
+            params['offset'] = data['load-next']
+        return self.playlist_result(entries,
+                                    playlist_title='%s activity' % channel_name)

From 58f6c2112d55cdd77ad76b323760bb934d7e7576 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 17 Jan 2021 14:07:56 +0100
Subject: [PATCH 130/860] [minds] improve extraction

---
 youtube_dl/extractor/extractors.py |   2 +-
 youtube_dl/extractor/minds.py      | 288 ++++++++++++++++-------------
 2 files changed, 161 insertions(+), 129 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 90012fc4f..29b0e615e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -653,8 +653,8 @@ from .microsoftvirtualacademy import (
 )
 from .minds import (
     MindsIE,
-    MindsActivityIE,
     MindsChannelIE,
+    MindsGroupIE,
 )
 from .ministrygrid import MinistryGridIE
 from .minoto import MinotoIE
diff --git a/youtube_dl/extractor/minds.py b/youtube_dl/extractor/minds.py
index 4523d0938..8e9f0f825 100644
--- a/youtube_dl/extractor/minds.py
+++ b/youtube_dl/extractor/minds.py
@@ -1,164 +1,196 @@
 # coding: utf-8
 from __future__ import unicode_literals
-import re
 
 from .common import InfoExtractor
 from ..compat import compat_str
-from ..utils import (int_or_none, sanitized_Request, str_or_none,
-                     unified_strdate)
+from ..utils import (
+    clean_html,
+    int_or_none,
+    str_or_none,
+    strip_or_none,
+)
 
 
-class MindsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?minds\.com/media/(?P<id>[0-9]+)'
-    _TEST = {
+class MindsBaseIE(InfoExtractor):
+    _VALID_URL_BASE = r'https?://(?:www\.)?minds\.com/'
+
+    def _call_api(self, path, video_id, resource, query=None):
+        api_url = 'https://www.minds.com/api/' + path
+        token = self._get_cookies(api_url).get('XSRF-TOKEN')
+        return self._download_json(
+            api_url, video_id, 'Downloading %s JSON metadata' % resource, headers={
+                'Referer': 'https://www.minds.com/',
+                'X-XSRF-TOKEN': token.value if token else '',
+            }, query=query)
+
+
+class MindsIE(MindsBaseIE):
+    IE_NAME = 'minds'
+    _VALID_URL = MindsBaseIE._VALID_URL_BASE + r'(?:media|newsfeed|archive/view)/(?P<id>[0-9]+)'
+    _TESTS = [{
         'url': 'https://www.minds.com/media/100000000000086822',
         'md5': '215a658184a419764852239d4970b045',
         'info_dict': {
             'id': '100000000000086822',
             'ext': 'mp4',
             'title': 'Minds intro sequence',
-            'thumbnail': 'https://cdn-cinemr.minds.com/cinemr_com/334128440657580032/thumbnail-00001.png',
-            'uploader_id': '100000000000000341',
-            'description': '<?xml encoding="utf-8" ?>',
+            'thumbnail': r're:https?://.+\.png',
+            'uploader_id': 'ottman',
             'upload_date': '20130524',
             'timestamp': 1369404826,
+            'uploader': 'Bill Ottman',
+            'view_count': int,
+            'like_count': int,
+            'dislike_count': int,
+            'tags': ['animation'],
+            'comment_count': int,
+            'license': 'attribution-cc',
         },
-        'params': {
-            'skip_download': True,
+    }, {
+        # entity.type == 'activity' and empty title
+        'url': 'https://www.minds.com/newsfeed/798025111988506624',
+        'md5': 'b2733a74af78d7fd3f541c4cbbaa5950',
+        'info_dict': {
+            'id': '798022190320226304',
+            'ext': 'mp4',
+            'title': '798022190320226304',
+            'uploader': 'ColinFlaherty',
+            'upload_date': '20180111',
+            'timestamp': 1515639316,
+            'uploader_id': 'ColinFlaherty',
         },
-    }
+    }, {
+        'url': 'https://www.minds.com/archive/view/715172106794442752',
+        'only_matching': True,
+    }, {
+        # youtube perma_url
+        'url': 'https://www.minds.com/newsfeed/1197131838022602752',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        video_api_url = 'https://www.minds.com/api/v1/media/%s' % video_id
-        token = self._get_cookies(url).get('XSRF-TOKEN')
-        headers = {
-            'authority': 'www.minds.com',
-            'referer': url,
-            'x-xsrf-token': token.value if token else '',
-        }
-        data = self._download_json(video_api_url, video_id, headers=headers,
-                                   query={'children': 'false'})
+        entity_id = self._match_id(url)
+        entity = self._call_api(
+            'v1/entities/entity/' + entity_id, entity_id, 'entity')['entity']
+        if entity.get('type') == 'activity':
+            if entity.get('custom_type') == 'video':
+                video_id = entity['entity_guid']
+            else:
+                return self.url_result(entity['perma_url'])
+        else:
+            assert(entity['subtype'] == 'video')
+            video_id = entity_id
+        # 1080p and webm formats available only on the sources array
+        video = self._call_api(
+            'v2/media/video/' + video_id, video_id, 'video')
+
         formats = []
-        owner = data.get('ownerObj', {})
+        for source in (video.get('sources') or []):
+            src = source.get('src')
+            if not src:
+                continue
+            formats.append({
+                'format_id': source.get('label'),
+                'height': int_or_none(source.get('size')),
+                'url': src,
+            })
+        self._sort_formats(formats)
 
-        transcodes = data.get('transcodes', {})
-        # These keys are the width so keep the highest width last
-        keys = sorted(transcodes.keys())
+        entity = video.get('entity') or entity
+        owner = entity.get('ownerObj') or {}
+        uploader_id = owner.get('username')
 
-        for format_id in keys:
-            is_numeric = re.match('^[0-9]+\.mp4', format_id)
-            video_url = transcodes[format_id]
-            info = {
-                'url': video_url,
-                'format_id': format_id,
-                'http_headers': headers,
-            }
-            if is_numeric:
-                info['width'] = int(format_id.split('.')[0])
-            formats.append(info)
+        tags = entity.get('tags')
+        if tags and isinstance(tags, compat_str):
+            tags = [tags]
 
-        uploader_id = str_or_none(owner.get('guid') or
-                                  data.get('owner_guid') or
-                                  owner.get('legacy_guid') or
-                                  owner.get('owner_guid'))
-        description = str_or_none(data.get('description'))
-        if description:
-            description = description.strip()
-        uploader_url = age_limit = thumbnail = None
-
-        if owner.get('username'):
-            uploader_url = 'https://www.minds.com/%s' % owner.get('username')
-        if data.get('mature') is True:
-            age_limit = 18
-
-        thumbnail_api_url = data.get('thumbnail_src')
-        if thumbnail_api_url:
-            req = sanitized_Request(thumbnail_api_url)
-            req.get_method = lambda: 'HEAD'
-            res = self._request_webpage(req, video_id)
-            if res.headers.get('content-type', '').startswith('image/'):
-                thumbnail = getattr(res, 'url', None)
-        tags = data.get('tags', '').strip()
-        if isinstance(tags, compat_str) and tags:
-            tags = [x.strip() for x in tags.split(',')]
-        else:
-            tags = None
-        category = data.get('category')
-        if isinstance(category, compat_str) and category:
-            category = [category]
-        else:
-            category = None
+        thumbnail = None
+        poster = video.get('poster') or entity.get('thumbnail_src')
+        if poster:
+            urlh = self._request_webpage(poster, video_id, fatal=False)
+            if urlh:
+                thumbnail = urlh.geturl()
 
         return {
             'id': video_id,
-            'title': data['title'],
+            'title': entity.get('title') or video_id,
             'formats': formats,
-            'description': description,
-            'license': str_or_none(data.get('license')),
-            'creator': str_or_none(owner.get('name') or owner.get('username')),
-            'release_date': unified_strdate(data.get('time_created')),
-            'timestamp': int_or_none(data.get('time_created')),
+            'description': clean_html(entity.get('description')) or None,
+            'license': str_or_none(entity.get('license')),
+            'timestamp': int_or_none(entity.get('time_created')),
+            'uploader': strip_or_none(owner.get('name')),
             'uploader_id': uploader_id,
-            'uploader_url': uploader_url,
-            'view_count': int_or_none(data.get('play:count')),
-            'like_count': int_or_none(data.get('thumbs:up:count')),
-            'dislike_count': int_or_none(data.get('thumbs:down:count')),
-            'average_rating': int_or_none(data.get('rating')),
-            'age_limit': age_limit,
-            'categories': [str_or_none(data.get('category'))],
+            'uploader_url': 'https://www.minds.com/' + uploader_id if uploader_id else None,
+            'view_count': int_or_none(entity.get('play:count')),
+            'like_count': int_or_none(entity.get('thumbs:up:count')),
+            'dislike_count': int_or_none(entity.get('thumbs:down:count')),
             'tags': tags,
-            # As of 20181020 the API is returning `false` for this value both
-            # at top level and within the entity.comments:count path. The only
-            # other way to get this is to fetch all comments and count.
-            'comment_count': int_or_none(data.get('comments:count')),
+            'comment_count': int_or_none(entity.get('comments:count')),
             'thumbnail': thumbnail,
         }
 
 
-class MindsActivityIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?minds\.com/newsfeed/(?P<id>[0-9]+)'
+class MindsFeedBaseIE(MindsBaseIE):
+    _PAGE_SIZE = 150
 
-    def _real_extract(self, url):
-        guid = self._match_id(url)
-        api_url = 'https://www.minds.com/api/v1/newsfeed/single/%s' % guid
-        token = self._get_cookies(url).get('XSRF-TOKEN')
-        headers = {
-            'authority': 'www.minds.com',
-            'referer': url,
-            'x-xsrf-token': token.value if token else '',
-        }
-        data = self._download_json(api_url, guid, headers=headers)
-        return self.url_result('https://www.minds.com/media/%s' % data['activity']['entity_guid'])
-
-
-class MindsChannelIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?minds\.com/(?!newsfeed|media|api)(?P<id>[^/]+)'
-
-    def _real_extract(self, url):
-        channel_name = self._match_id(url)
-        api_url = 'https://www.minds.com/api/v1/channel/%s' % channel_name
-        token = self._get_cookies(url).get('XSRF-TOKEN')
-        headers = {
-            'authority': 'www.minds.com',
-            'referer': url,
-            'x-xsrf-token': token.value if token else '',
-        }
-        data = self._download_json(api_url, channel_name, headers=headers)
-        channel = data.get('channel', {})
-        params = {'limit': 12, 'offset': ''}
-        api_url = 'https://www.minds.com/api/v1/newsfeed/personal/%s' % channel['guid']
-        entries = []
+    def _entries(self, feed_id):
+        query = {'limit': self._PAGE_SIZE, 'sync': 1}
+        i = 1
         while True:
-            data = self._download_json(api_url, channel['guid'],
-                                       headers=headers, query=params)
-            activity = data.get('activity', [])
-            if len(activity) == 0 or not data.get('load-next'):
-                break
-            for info in activity:
-                if info.get('custom_type') != 'video':
+            data = self._call_api(
+                'v2/feeds/container/%s/videos' % feed_id,
+                feed_id, 'page %s' % i, query)
+            entities = data.get('entities') or []
+            for entity in entities:
+                guid = entity.get('guid')
+                if not guid:
                     continue
-                entries.append(self.url_result('https://www.minds.com/media/%s' % info['entity_guid']))
-            params['offset'] = data['load-next']
-        return self.playlist_result(entries,
-                                    playlist_title='%s activity' % channel_name)
+                yield self.url_result(
+                    'https://www.minds.com/newsfeed/' + guid,
+                    MindsIE.ie_key(), guid)
+            query['from_timestamp'] = data['load-next']
+            if not (query['from_timestamp'] and len(entities) == self._PAGE_SIZE):
+                break
+            i += 1
+
+    def _real_extract(self, url):
+        feed_id = self._match_id(url)
+        feed = self._call_api(
+            'v1/%s/%s' % (self._FEED_PATH, feed_id),
+            feed_id, self._FEED_TYPE)[self._FEED_TYPE]
+
+        return self.playlist_result(
+            self._entries(feed['guid']), feed_id,
+            strip_or_none(feed.get('name')),
+            feed.get('briefdescription'))
+
+
+class MindsChannelIE(MindsFeedBaseIE):
+    _FEED_TYPE = 'channel'
+    IE_NAME = 'minds:' + _FEED_TYPE
+    _VALID_URL = MindsBaseIE._VALID_URL_BASE + r'(?!(?:newsfeed|media|api|archive|groups)/)(?P<id>[^/?&#]+)'
+    _FEED_PATH = 'channel'
+    _TEST = {
+        'url': 'https://www.minds.com/ottman',
+        'info_dict': {
+            'id': 'ottman',
+            'title': 'Bill Ottman',
+            'description': 'Co-creator & CEO @minds',
+        },
+        'playlist_mincount': 54,
+    }
+
+
+class MindsGroupIE(MindsFeedBaseIE):
+    _FEED_TYPE = 'group'
+    IE_NAME = 'minds:' + _FEED_TYPE
+    _VALID_URL = MindsBaseIE._VALID_URL_BASE + r'groups/profile/(?P<id>[0-9]+)'
+    _FEED_PATH = 'groups/group'
+    _TEST = {
+        'url': 'https://www.minds.com/groups/profile/785582576369672204/feed/videos',
+        'info_dict': {
+            'id': '785582576369672204',
+            'title': 'Cooking Videos',
+        },
+        'playlist_mincount': 1,
+    }

From 26499ba823ecba99e18cf5cc76e001f4bbbe4293 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 17 Jan 2021 17:35:10 +0100
Subject: [PATCH 131/860] [aljazeera] fix extraction(closes #20911)(closes
 #27779)

---
 youtube_dl/extractor/aljazeera.py | 41 ++++++++++++++++++++++++-------
 1 file changed, 32 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/aljazeera.py b/youtube_dl/extractor/aljazeera.py
index c68be3134..c4f915a3c 100644
--- a/youtube_dl/extractor/aljazeera.py
+++ b/youtube_dl/extractor/aljazeera.py
@@ -1,13 +1,16 @@
 from __future__ import unicode_literals
 
+import json
+import re
+
 from .common import InfoExtractor
 
 
 class AlJazeeraIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?aljazeera\.com/(?:programmes|video)/.*?/(?P<id>[^/]+)\.html'
+    _VALID_URL = r'https?://(?:www\.)?aljazeera\.com/(?P<type>program/[^/]+|(?:feature|video)s)/\d{4}/\d{1,2}/\d{1,2}/(?P<id>[^/?&#]+)'
 
     _TESTS = [{
-        'url': 'http://www.aljazeera.com/programmes/the-slum/2014/08/deliverance-201482883754237240.html',
+        'url': 'https://www.aljazeera.com/program/episode/2014/9/19/deliverance',
         'info_dict': {
             'id': '3792260579001',
             'ext': 'mp4',
@@ -20,14 +23,34 @@ class AlJazeeraIE(InfoExtractor):
         'add_ie': ['BrightcoveNew'],
         'skip': 'Not accessible from Travis CI server',
     }, {
-        'url': 'http://www.aljazeera.com/video/news/2017/05/sierra-leone-709-carat-diamond-auctioned-170511100111930.html',
+        'url': 'https://www.aljazeera.com/videos/2017/5/11/sierra-leone-709-carat-diamond-to-be-auctioned-off',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.aljazeera.com/features/2017/8/21/transforming-pakistans-buses-into-art',
         'only_matching': True,
     }]
-    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/665003303001/default_default/index.html?videoId=%s'
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
-        program_name = self._match_id(url)
-        webpage = self._download_webpage(url, program_name)
-        brightcove_id = self._search_regex(
-            r'RenderPagesVideo\(\'(.+?)\'', webpage, 'brightcove id')
-        return self.url_result(self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id, 'BrightcoveNew', brightcove_id)
+        post_type, name = re.match(self._VALID_URL, url).groups()
+        post_type = {
+            'features': 'post',
+            'program': 'episode',
+            'videos': 'video',
+        }[post_type.split('/')[0]]
+        video = self._download_json(
+            'https://www.aljazeera.com/graphql', name, query={
+                'operationName': 'SingleArticleQuery',
+                'variables': json.dumps({
+                    'name': name,
+                    'postType': post_type,
+                }),
+            }, headers={
+                'wp-site': 'aje',
+            })['data']['article']['video']
+        video_id = video['id']
+        account_id = video.get('accountId') or '665003303001'
+        player_id = video.get('playerId') or 'BkeSH5BDb'
+        return self.url_result(
+            self.BRIGHTCOVE_URL_TEMPLATE % (account_id, player_id, video_id),
+            'BrightcoveNew', video_id)

From f9201cef58c2ef6bd99315ea06b57d0c69a83f59 Mon Sep 17 00:00:00 2001
From: Brian Marks <bm1549@users.noreply.github.com>
Date: Mon, 18 Jan 2021 09:47:06 -0500
Subject: [PATCH 132/860] [americastestkitchen] Improve metadata extraction for
 ATK episodes (#27860)

---
 youtube_dl/extractor/americastestkitchen.py | 30 +++++++++++++++++++--
 1 file changed, 28 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/americastestkitchen.py b/youtube_dl/extractor/americastestkitchen.py
index e20f00fc3..7d2c375c4 100644
--- a/youtube_dl/extractor/americastestkitchen.py
+++ b/youtube_dl/extractor/americastestkitchen.py
@@ -6,8 +6,10 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
+    int_or_none,
     try_get,
     unified_strdate,
+    unified_timestamp,
 )
 
 
@@ -22,8 +24,8 @@ class AmericasTestKitchenIE(InfoExtractor):
             'ext': 'mp4',
             'description': 'md5:64e606bfee910627efc4b5f050de92b3',
             'thumbnail': r're:^https?://',
-            'timestamp': 1523664000,
-            'upload_date': '20180414',
+            'timestamp': 1523318400,
+            'upload_date': '20180410',
             'release_date': '20180410',
             'series': "America's Test Kitchen",
             'season_number': 18,
@@ -33,6 +35,27 @@ class AmericasTestKitchenIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # Metadata parsing behaves differently for newer episodes (705) as opposed to older episodes (582 above)
+        'url': 'https://www.americastestkitchen.com/episode/705-simple-chicken-dinner',
+        'md5': '06451608c57651e985a498e69cec17e5',
+        'info_dict': {
+            'id': '5fbe8c61bda2010001c6763b',
+            'title': 'Simple Chicken Dinner',
+            'ext': 'mp4',
+            'description': 'md5:eb68737cc2fd4c26ca7db30139d109e7',
+            'thumbnail': r're:^https?://',
+            'timestamp': 1610755200,
+            'upload_date': '20210116',
+            'release_date': '20210116',
+            'series': "America's Test Kitchen",
+            'season_number': 21,
+            'episode': 'Simple Chicken Dinner',
+            'episode_number': 3,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'https://www.americastestkitchen.com/videos/3420-pan-seared-salmon',
         'only_matching': True,
@@ -60,7 +83,10 @@ class AmericasTestKitchenIE(InfoExtractor):
             'url': 'https://player.zype.com/embed/%s.js?api_key=jZ9GUhRmxcPvX7M3SlfejB6Hle9jyHTdk2jVxG7wOHPLODgncEKVdPYBhuz9iWXQ' % video['zypeId'],
             'ie_key': 'Zype',
             'description': clean_html(video.get('description')),
+            'timestamp': unified_timestamp(video.get('publishDate')),
             'release_date': unified_strdate(video.get('publishDate')),
+            'episode_number': int_or_none(episode.get('number')),
+            'season_number': int_or_none(episode.get('season')),
             'series': try_get(episode, lambda x: x['show']['title']),
             'episode': episode.get('title'),
         }

From 1dd12708c2042e5d78887e4c5026d6196cc02bb2 Mon Sep 17 00:00:00 2001
From: DrWursterich <31037782+DrWursterich@users.noreply.github.com>
Date: Tue, 19 Jan 2021 10:21:37 +0100
Subject: [PATCH 133/860] [9gag] Fix Extraction (#23022)

---
 youtube_dl/extractor/ninegag.py | 200 +++++++++++++++++++-------------
 1 file changed, 122 insertions(+), 78 deletions(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index dc6a27d36..3753bc0a2 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -3,102 +3,146 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import str_to_int
+from ..utils import (
+    determine_ext,
+    url_or_none,
+    int_or_none,
+    float_or_none,
+    ExtractorError
+)
 
 
 class NineGagIE(InfoExtractor):
     IE_NAME = '9gag'
-    _VALID_URL = r'https?://(?:www\.)?9gag(?:\.com/tv|\.tv)/(?:p|embed)/(?P<id>[a-zA-Z0-9]+)(?:/(?P<display_id>[^?#/]+))?'
+    _VALID_URL = r'https?://(?:www\.)?9gag\.com/gag/(?P<id>[a-zA-Z0-9]+)'
 
     _TESTS = [{
-        'url': 'http://9gag.com/tv/p/Kk2X5/people-are-awesome-2013-is-absolutely-awesome',
+        'url': 'https://9gag.com/gag/an5Qz5b',
         'info_dict': {
-            'id': 'kXzwOKyGlSA',
-            'ext': 'mp4',
-            'description': 'This 3-minute video will make you smile and then make you feel untalented and insignificant. Anyway, you should share this awesomeness. (Thanks, Dino!)',
-            'title': '\"People Are Awesome 2013\" Is Absolutely Awesome',
-            'uploader_id': 'UCdEH6EjDKwtTe-sO2f0_1XA',
-            'uploader': 'CompilationChannel',
-            'upload_date': '20131110',
-            'view_count': int,
-        },
-        'add_ie': ['Youtube'],
+            'id': 'an5Qz5b',
+            'ext': 'webm',
+            'title': 'Dogs playing tetherball',
+            'upload_date': '20191108',
+            'timestamp': 1573243994,
+            'age_limit': 0,
+            'categories': [
+                'Wholesome'
+            ],
+            'tags': [
+                'Dog'
+            ]
+        }
     }, {
-        'url': 'http://9gag.com/tv/p/aKolP3',
+        'url': 'https://9gag.com/gag/ae5Ag7B',
         'info_dict': {
-            'id': 'aKolP3',
-            'ext': 'mp4',
-            'title': 'This Guy Travelled 11 countries In 44 days Just To Make This Amazing Video',
-            'description': "I just saw more in 1 minute than I've seen in 1 year. This guy's video is epic!!",
-            'uploader_id': 'rickmereki',
-            'uploader': 'Rick Mereki',
-            'upload_date': '20110803',
-            'view_count': int,
-        },
-        'add_ie': ['Vimeo'],
-    }, {
-        'url': 'http://9gag.com/tv/p/KklwM',
-        'only_matching': True,
-    }, {
-        'url': 'http://9gag.tv/p/Kk2X5',
-        'only_matching': True,
-    }, {
-        'url': 'http://9gag.com/tv/embed/a5Dmvl',
-        'only_matching': True,
+            'id': 'ae5Ag7B',
+            'ext': 'webm',
+            'title': 'Capybara Agility Training',
+            'upload_date': '20191108',
+            'timestamp': 1573237208,
+            'age_limit': 0,
+            'categories': [
+                'Awesome'
+            ],
+            'tags': [
+                'Weimaraner',
+                'American Pit Bull Terrier'
+            ]
+        }
     }]
 
-    _EXTERNAL_VIDEO_PROVIDER = {
-        '1': {
-            'url': '%s',
-            'ie_key': 'Youtube',
-        },
-        '2': {
-            'url': 'http://player.vimeo.com/video/%s',
-            'ie_key': 'Vimeo',
-        },
-        '3': {
-            'url': 'http://instagram.com/p/%s',
-            'ie_key': 'Instagram',
-        },
-        '4': {
-            'url': 'http://vine.co/v/%s',
-            'ie_key': 'Vine',
-        },
+    _EXTERNAL_VIDEO_PROVIDERS = {
+        'Youtube': 'https://youtube.com/watch?v=%s'
     }
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id') or video_id
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        rawJsonData = self._search_regex(
+            r'window._config\s*=\s*JSON.parse\(["\']({.+?})["\']\);',
+            webpage,
+            'data')
+        rawJsonData = rawJsonData.replace('\\"', '"').replace('\\\\/', '/')
+        data = self._parse_json(rawJsonData, video_id)['data']['post']
 
-        webpage = self._download_webpage(url, display_id)
+        if data['type'] == 'Video':
+            vid = data['video']['id']
+            ie_key = data['video']['source'].capitalize()
+            return {
+                '_type': 'url_transparent',
+                'url': self._EXTERNAL_VIDEO_PROVIDERS[ie_key] % vid,
+                'ie_key': ie_key,
+                'id': vid,
+                'duration': data['video'].get('duration'),
+                'start_time': data['video'].get('startTs')
+            }
 
-        post_view = self._parse_json(
-            self._search_regex(
-                r'var\s+postView\s*=\s*new\s+app\.PostView\({\s*post:\s*({.+?})\s*,\s*posts:\s*prefetchedCurrentPost',
-                webpage, 'post view'),
-            display_id)
+        if data['type'] == 'EmbedVideo':
+            vid = data['video']['id']
+            ie_key = data['video']['source'].capitalize()
+            return {
+                '_type': 'url_transparent',
+                'url': data['video']['embedUrl'],
+                #'ie_key': vid,
+                'start_time': data['video'].get('startTs')
+            }
 
-        ie_key = None
-        source_url = post_view.get('sourceUrl')
-        if not source_url:
-            external_video_id = post_view['videoExternalId']
-            external_video_provider = post_view['videoExternalProvider']
-            source_url = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['url'] % external_video_id
-            ie_key = self._EXTERNAL_VIDEO_PROVIDER[external_video_provider]['ie_key']
-        title = post_view['title']
-        description = post_view.get('description')
-        view_count = str_to_int(post_view.get('externalView'))
-        thumbnail = post_view.get('thumbnail_700w') or post_view.get('ogImageUrl') or post_view.get('thumbnail_300w')
+        if data['type'] != 'Animated':
+            raise ExtractorError(
+                'The given url does not contain a video',
+                expected=True)
+
+        duration = None
+        formats = []
+        thumbnails = []
+        for key in data['images']:
+            image = data['images'][key]
+            if 'duration' in image and duration is None:
+                duration = int_or_none(image['duration'])
+            url = url_or_none(image.get('url'))
+            if url == None:
+                continue
+            ext = determine_ext(url)
+            if ext == 'jpg' or ext == 'png':
+                thumbnail = {
+                    'url': url,
+                    'width': float_or_none(image.get('width')),
+                    'height': float_or_none(image.get('height'))
+                }
+                thumbnails.append(thumbnail)
+            elif ext == 'webm' or ext == 'mp4':
+                formats.append({
+                    'format_id': re.sub(r'.*_([^\.]+).(.*)', r'\1_\2', url),
+                    'ext': ext,
+                    'url': url,
+                    'width': float_or_none(image.get('width')),
+                    'height': float_or_none(image.get('height'))
+                })
+        section = None
+        postSection = data.get('postSection')
+        if postSection != None and 'name' in postSection:
+            section = re.sub(r'\\[^\\]{5}', '', postSection['name'])
+        age_limit = int_or_none(data.get('nsfw'))
+        if age_limit != None:
+            age_limit = age_limit * 18
+        tags = None
+        if 'tags' in data:
+            tags = []
+            for tag in data.get('tags') or []:
+                tags.append(tag.get('key'))
 
         return {
-            '_type': 'url_transparent',
-            'url': source_url,
-            'ie_key': ie_key,
             'id': video_id,
-            'display_id': display_id,
-            'title': title,
-            'description': description,
-            'view_count': view_count,
-            'thumbnail': thumbnail,
+            'title': data['title'],
+            'timestamp': int_or_none(data.get('creationTs')),
+            'duration': duration,
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'like_count': int_or_none(data.get('upVoteCount')),
+            'dislike_count': int_or_none(data.get('downVoteCount')),
+            'comment_count': int_or_none(data.get('commentsCount')),
+            'age_limit': age_limit,
+            'categories': [section],
+            'tags': tags,
+            'is_live': False
         }

From 54856480d7bac670c9d571d4191f5f35aadc5270 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 19 Jan 2021 10:23:02 +0100
Subject: [PATCH 134/860] [ninegag] improve extraction

---
 youtube_dl/extractor/ninegag.py | 189 ++++++++++++++------------------
 1 file changed, 83 insertions(+), 106 deletions(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 3753bc0a2..440f865bc 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -1,148 +1,125 @@
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
-    url_or_none,
+    ExtractorError,
     int_or_none,
-    float_or_none,
-    ExtractorError
+    try_get,
+    url_or_none,
 )
 
 
 class NineGagIE(InfoExtractor):
     IE_NAME = '9gag'
-    _VALID_URL = r'https?://(?:www\.)?9gag\.com/gag/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?9gag\.com/gag/(?P<id>[^/?&#]+)'
 
-    _TESTS = [{
-        'url': 'https://9gag.com/gag/an5Qz5b',
-        'info_dict': {
-            'id': 'an5Qz5b',
-            'ext': 'webm',
-            'title': 'Dogs playing tetherball',
-            'upload_date': '20191108',
-            'timestamp': 1573243994,
-            'age_limit': 0,
-            'categories': [
-                'Wholesome'
-            ],
-            'tags': [
-                'Dog'
-            ]
-        }
-    }, {
+    _TEST = {
         'url': 'https://9gag.com/gag/ae5Ag7B',
         'info_dict': {
             'id': 'ae5Ag7B',
-            'ext': 'webm',
+            'ext': 'mp4',
             'title': 'Capybara Agility Training',
             'upload_date': '20191108',
             'timestamp': 1573237208,
-            'age_limit': 0,
-            'categories': [
-                'Awesome'
-            ],
-            'tags': [
-                'Weimaraner',
-                'American Pit Bull Terrier'
-            ]
+            'categories': ['Awesome'],
+            'tags': ['Weimaraner', 'American Pit Bull Terrier'],
+            'duration': 44,
+            'like_count': int,
+            'dislike_count': int,
+            'comment_count': int,
         }
-    }]
-
-    _EXTERNAL_VIDEO_PROVIDERS = {
-        'Youtube': 'https://youtube.com/watch?v=%s'
     }
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        rawJsonData = self._search_regex(
-            r'window._config\s*=\s*JSON.parse\(["\']({.+?})["\']\);',
-            webpage,
-            'data')
-        rawJsonData = rawJsonData.replace('\\"', '"').replace('\\\\/', '/')
-        data = self._parse_json(rawJsonData, video_id)['data']['post']
+        post_id = self._match_id(url)
+        post = self._download_json(
+            'https://9gag.com/v1/post', post_id, query={
+                'id': post_id
+            })['data']['post']
 
-        if data['type'] == 'Video':
-            vid = data['video']['id']
-            ie_key = data['video']['source'].capitalize()
-            return {
-                '_type': 'url_transparent',
-                'url': self._EXTERNAL_VIDEO_PROVIDERS[ie_key] % vid,
-                'ie_key': ie_key,
-                'id': vid,
-                'duration': data['video'].get('duration'),
-                'start_time': data['video'].get('startTs')
-            }
-
-        if data['type'] == 'EmbedVideo':
-            vid = data['video']['id']
-            ie_key = data['video']['source'].capitalize()
-            return {
-                '_type': 'url_transparent',
-                'url': data['video']['embedUrl'],
-                #'ie_key': vid,
-                'start_time': data['video'].get('startTs')
-            }
-
-        if data['type'] != 'Animated':
+        if post.get('type') != 'Animated':
             raise ExtractorError(
                 'The given url does not contain a video',
                 expected=True)
 
+        title = post['title']
+
         duration = None
         formats = []
         thumbnails = []
-        for key in data['images']:
-            image = data['images'][key]
-            if 'duration' in image and duration is None:
-                duration = int_or_none(image['duration'])
-            url = url_or_none(image.get('url'))
-            if url == None:
+        for key, image in (post.get('images') or {}).items():
+            image_url = url_or_none(image.get('url'))
+            if not image_url:
                 continue
-            ext = determine_ext(url)
-            if ext == 'jpg' or ext == 'png':
-                thumbnail = {
-                    'url': url,
-                    'width': float_or_none(image.get('width')),
-                    'height': float_or_none(image.get('height'))
-                }
-                thumbnails.append(thumbnail)
-            elif ext == 'webm' or ext == 'mp4':
-                formats.append({
-                    'format_id': re.sub(r'.*_([^\.]+).(.*)', r'\1_\2', url),
+            ext = determine_ext(image_url)
+            image_id = key.strip('image')
+            common = {
+                'url': image_url,
+                'width': int_or_none(image.get('width')),
+                'height': int_or_none(image.get('height')),
+            }
+            if ext in ('jpg', 'png'):
+                webp_url = image.get('webpUrl')
+                if webp_url:
+                    t = common.copy()
+                    t.update({
+                        'id': image_id + '-webp',
+                        'url': webp_url,
+                    })
+                    thumbnails.append(t)
+                common.update({
+                    'id': image_id,
                     'ext': ext,
-                    'url': url,
-                    'width': float_or_none(image.get('width')),
-                    'height': float_or_none(image.get('height'))
                 })
-        section = None
-        postSection = data.get('postSection')
-        if postSection != None and 'name' in postSection:
-            section = re.sub(r'\\[^\\]{5}', '', postSection['name'])
-        age_limit = int_or_none(data.get('nsfw'))
-        if age_limit != None:
-            age_limit = age_limit * 18
+                thumbnails.append(common)
+            elif ext in ('webm', 'mp4'):
+                if not duration:
+                    duration = int_or_none(image.get('duration'))
+                common['acodec'] = 'none' if image.get('hasAudio') == 0 else None
+                for vcodec in ('vp8', 'vp9', 'h265'):
+                    c_url = image.get(vcodec + 'Url')
+                    if not c_url:
+                        continue
+                    c_f = common.copy()
+                    c_f.update({
+                        'format_id': image_id + '-' + vcodec,
+                        'url': c_url,
+                        'vcodec': vcodec,
+                    })
+                    formats.append(c_f)
+                common.update({
+                    'ext': ext,
+                    'format_id': image_id,
+                })
+                formats.append(common)
+        self._sort_formats(formats)
+
+        section = try_get(post, lambda x: x['postSection']['name'])
+
         tags = None
-        if 'tags' in data:
+        post_tags = post.get('tags')
+        if post_tags:
             tags = []
-            for tag in data.get('tags') or []:
-                tags.append(tag.get('key'))
+            for tag in post_tags:
+                tag_key = tag.get('key')
+                if not tag_key:
+                    continue
+                tags.append(tag_key)
+
+        get_count = lambda x: int_or_none(post.get(x + 'Count'))
 
         return {
-            'id': video_id,
-            'title': data['title'],
-            'timestamp': int_or_none(data.get('creationTs')),
+            'id': post_id,
+            'title': title,
+            'timestamp': int_or_none(post.get('creationTs')),
             'duration': duration,
             'formats': formats,
             'thumbnails': thumbnails,
-            'like_count': int_or_none(data.get('upVoteCount')),
-            'dislike_count': int_or_none(data.get('downVoteCount')),
-            'comment_count': int_or_none(data.get('commentsCount')),
-            'age_limit': age_limit,
-            'categories': [section],
+            'like_count': get_count('upVote'),
+            'dislike_count': get_count('downVote'),
+            'comment_count': get_count('comments'),
+            'age_limit': 18 if post.get('nsfw') == 1 else None,
+            'categories': [section] if section else None,
             'tags': tags,
-            'is_live': False
         }

From 015e19b350a4bb7868008fb9df9092f4a60a5f00 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 19 Jan 2021 11:54:39 +0100
Subject: [PATCH 135/860] [lbry] unescape lbry uri(closes #27872)

---
 youtube_dl/extractor/lbry.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lbry.py b/youtube_dl/extractor/lbry.py
index 41cc245eb..413215a99 100644
--- a/youtube_dl/extractor/lbry.py
+++ b/youtube_dl/extractor/lbry.py
@@ -5,7 +5,10 @@ import functools
 import json
 
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import (
+    compat_str,
+    compat_urllib_parse_unquote,
+)
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -131,6 +134,9 @@ class LBRYIE(LBRYBaseIE):
     }, {
         'url': 'https://lbry.tv/$/download/Episode-1/e7d93d772bd87e2b62d5ab993c1c3ced86ebb396',
         'only_matching': True,
+    }, {
+        'url': 'https://lbry.tv/@lacajadepandora:a/TRUMP-EST%C3%81-BIEN-PUESTO-con-Pilar-Baselga,-Carlos-Senra,-Luis-Palacios-(720p_30fps_H264-192kbit_AAC):1',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -139,6 +145,7 @@ class LBRYIE(LBRYBaseIE):
             display_id = display_id.split('/', 2)[-1].replace('/', ':')
         else:
             display_id = display_id.replace(':', '#')
+        display_id = compat_urllib_parse_unquote(display_id)
         uri = 'lbry://' + display_id
         result = self._resolve_url(uri, display_id, 'stream')
         result_value = result['value']

From bc7c8f3d4ea6cb98fa62a5fe457046dd4e9b5379 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 19 Jan 2021 14:47:39 +0100
Subject: [PATCH 136/860] [yahoo] fix single video extraction

---
 youtube_dl/extractor/yahoo.py | 80 +++++++++++++++++------------------
 1 file changed, 40 insertions(+), 40 deletions(-)

diff --git a/youtube_dl/extractor/yahoo.py b/youtube_dl/extractor/yahoo.py
index e4615376c..a17b10d6e 100644
--- a/youtube_dl/extractor/yahoo.py
+++ b/youtube_dl/extractor/yahoo.py
@@ -177,46 +177,9 @@ class YahooIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    def _real_extract(self, url):
-        url, country, display_id = re.match(self._VALID_URL, url).groups()
-        if not country:
-            country = 'us'
-        else:
-            country = country.split('-')[0]
-        api_base = 'https://%s.yahoo.com/_td/api/resource/' % country
-
-        for i, uuid in enumerate(['url=' + url, 'ymedia-alias=' + display_id]):
-            content = self._download_json(
-                api_base + 'content;getDetailView=true;uuids=["%s"]' % uuid,
-                display_id, 'Downloading content JSON metadata', fatal=i == 1)
-            if content:
-                item = content['items'][0]
-                break
-
-        if item.get('type') != 'video':
-            entries = []
-
-            cover = item.get('cover') or {}
-            if cover.get('type') == 'yvideo':
-                cover_url = cover.get('url')
-                if cover_url:
-                    entries.append(self.url_result(
-                        cover_url, 'Yahoo', cover.get('uuid')))
-
-            for e in item.get('body', []):
-                if e.get('type') == 'videoIframe':
-                    iframe_url = e.get('url')
-                    if not iframe_url:
-                        continue
-                    entries.append(self.url_result(iframe_url))
-
-            return self.playlist_result(
-                entries, item.get('uuid'),
-                item.get('title'), item.get('summary'))
-
-        video_id = item['uuid']
+    def _extract_yahoo_video(self, video_id, country):
         video = self._download_json(
-            api_base + 'VideoService.videos;view=full;video_ids=["%s"]' % video_id,
+            'https://%s.yahoo.com/_td/api/resource/VideoService.videos;view=full;video_ids=["%s"]' % (country, video_id),
             video_id, 'Downloading video JSON metadata')[0]
         title = video['title']
 
@@ -298,7 +261,6 @@ class YahooIE(InfoExtractor):
             'id': video_id,
             'title': self._live_title(title) if is_live else title,
             'formats': formats,
-            'display_id': display_id,
             'thumbnails': thumbnails,
             'description': clean_html(video.get('description')),
             'timestamp': parse_iso8601(video.get('publish_time')),
@@ -311,6 +273,44 @@ class YahooIE(InfoExtractor):
             'episode_number': int_or_none(series_info.get('episode_number')),
         }
 
+    def _real_extract(self, url):
+        url, country, display_id = re.match(self._VALID_URL, url).groups()
+        if not country:
+            country = 'us'
+        else:
+            country = country.split('-')[0]
+
+        item = self._download_json(
+            'https://%s.yahoo.com/caas/content/article' % country, display_id,
+            'Downloading content JSON metadata', query={
+                'url': url
+            })['items'][0]['data']['partnerData']
+
+        if item.get('type') != 'video':
+            entries = []
+
+            cover = item.get('cover') or {}
+            if cover.get('type') == 'yvideo':
+                cover_url = cover.get('url')
+                if cover_url:
+                    entries.append(self.url_result(
+                        cover_url, 'Yahoo', cover.get('uuid')))
+
+            for e in (item.get('body') or []):
+                if e.get('type') == 'videoIframe':
+                    iframe_url = e.get('url')
+                    if not iframe_url:
+                        continue
+                    entries.append(self.url_result(iframe_url))
+
+            return self.playlist_result(
+                entries, item.get('uuid'),
+                item.get('title'), item.get('summary'))
+
+        info = self._extract_yahoo_video(item['uuid'], country)
+        info['display_id'] = display_id
+        return info
+
 
 class YahooSearchIE(SearchInfoExtractor):
     IE_DESC = 'Yahoo screen search'

From b79977fb6b1b2cea0231e5e25e201db05c0d1dba Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 19 Jan 2021 14:49:45 +0100
Subject: [PATCH 137/860] [aol] add support for yahoo videos(closes #26650)

---
 youtube_dl/extractor/aol.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/aol.py b/youtube_dl/extractor/aol.py
index e87994a6a..f6ecb8438 100644
--- a/youtube_dl/extractor/aol.py
+++ b/youtube_dl/extractor/aol.py
@@ -3,7 +3,7 @@ from __future__ import unicode_literals
 
 import re
 
-from .common import InfoExtractor
+from .yahoo import YahooIE
 from ..compat import (
     compat_parse_qs,
     compat_urllib_parse_urlparse,
@@ -15,9 +15,9 @@ from ..utils import (
 )
 
 
-class AolIE(InfoExtractor):
+class AolIE(YahooIE):
     IE_NAME = 'aol.com'
-    _VALID_URL = r'(?:aol-video:|https?://(?:www\.)?aol\.(?:com|ca|co\.uk|de|jp)/video/(?:[^/]+/)*)(?P<id>[0-9a-f]+)'
+    _VALID_URL = r'(?:aol-video:|https?://(?:www\.)?aol\.(?:com|ca|co\.uk|de|jp)/video/(?:[^/]+/)*)(?P<id>\d{9}|[0-9a-f]{24}|[0-9a-f]{8}-(?:[0-9a-f]{4}-){3}[0-9a-f]{12})'
 
     _TESTS = [{
         # video with 5min ID
@@ -76,10 +76,16 @@ class AolIE(InfoExtractor):
     }, {
         'url': 'https://www.aol.jp/video/playlist/5a28e936a1334d000137da0c/5a28f3151e642219fde19831/',
         'only_matching': True,
+    }, {
+        # Yahoo video
+        'url': 'https://www.aol.com/video/play/991e6700-ac02-11ea-99ff-357400036f61/24bbc846-3e30-3c46-915e-fe8ccd7fcc46/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        if '-' in video_id:
+            return self._extract_yahoo_video(video_id, 'us')
 
         response = self._download_json(
             'https://feedapi.b2c.on.aol.com/v1.0/app/videos/aolon/%s/details' % video_id,

From 62acf5fa2c23d2eb52fb1dd07804352116bba12f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 19 Jan 2021 22:43:59 +0100
Subject: [PATCH 138/860] [trovo] Add new extractor(closes #26125)

---
 youtube_dl/extractor/extractors.py |   4 +
 youtube_dl/extractor/trovo.py      | 193 +++++++++++++++++++++++++++++
 2 files changed, 197 insertions(+)
 create mode 100644 youtube_dl/extractor/trovo.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 29b0e615e..536b184bc 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1238,6 +1238,10 @@ from .toutv import TouTvIE
 from .toypics import ToypicsUserIE, ToypicsIE
 from .traileraddict import TrailerAddictIE
 from .trilulilu import TriluliluIE
+from .trovo import (
+    TrovoIE,
+    TrovoVodIE,
+)
 from .trunews import TruNewsIE
 from .trutv import TruTVIE
 from .tube8 import Tube8IE
diff --git a/youtube_dl/extractor/trovo.py b/youtube_dl/extractor/trovo.py
new file mode 100644
index 000000000..43745213d
--- /dev/null
+++ b/youtube_dl/extractor/trovo.py
@@ -0,0 +1,193 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    str_or_none,
+    try_get,
+)
+
+
+class TrovoBaseIE(InfoExtractor):
+    _VALID_URL_BASE = r'https?://(?:www\.)?trovo\.live/'
+
+    def _extract_streamer_info(self, data):
+        streamer_info = data.get('streamerInfo') or {}
+        username = streamer_info.get('userName')
+        return {
+            'uploader': streamer_info.get('nickName'),
+            'uploader_id': str_or_none(streamer_info.get('uid')),
+            'uploader_url': 'https://trovo.live/' + username if username else None,
+        }
+
+
+class TrovoIE(TrovoBaseIE):
+    _VALID_URL = TrovoBaseIE._VALID_URL_BASE + r'(?!(?:clip|video)/)(?P<id>[^/?&#]+)'
+
+    def _real_extract(self, url):
+        username = self._match_id(url)
+        live_info = self._download_json(
+            'https://gql.trovo.live/', username, query={
+                'query': '''{
+  getLiveInfo(params: {userName: "%s"}) {
+    isLive
+    programInfo	{
+      coverUrl
+      id
+      streamInfo {
+        desc
+        playUrl
+      }
+      title
+    }
+    streamerInfo {
+        nickName
+        uid
+        userName
+    }
+  }
+}''' % username,
+            })['data']['getLiveInfo']
+        if live_info.get('isLive') == 0:
+            raise ExtractorError('%s is offline' % username, expected=True)
+        program_info = live_info['programInfo']
+        program_id = program_info['id']
+        title = self._live_title(program_info['title'])
+
+        formats = []
+        for stream_info in (program_info.get('streamInfo') or []):
+            play_url = stream_info.get('playUrl')
+            if not play_url:
+                continue
+            format_id = stream_info.get('desc')
+            formats.append({
+                'format_id': format_id,
+                'height': int_or_none(format_id[:-1]) if format_id else None,
+                'url': play_url,
+            })
+        self._sort_formats(formats)
+
+        info = {
+            'id': program_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': program_info.get('coverUrl'),
+            'is_live': True,
+        }
+        info.update(self._extract_streamer_info(live_info))
+        return info
+
+
+class TrovoVodIE(TrovoBaseIE):
+    _VALID_URL = TrovoBaseIE._VALID_URL_BASE + r'(?:clip|video)/(?P<id>[^/?&#]+)'
+    _TESTS = [{
+        'url': 'https://trovo.live/video/ltv-100095501_100095501_1609596043',
+        'info_dict': {
+            'id': 'ltv-100095501_100095501_1609596043',
+            'ext': 'mp4',
+            'title': 'Spontaner 12 Stunden Stream! - Ok Boomer!',
+            'uploader': 'Exsl',
+            'timestamp': 1609640305,
+            'upload_date': '20210103',
+            'uploader_id': '100095501',
+            'duration': 43977,
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+            'comments': 'mincount:8',
+            'categories': ['Grand Theft Auto V'],
+        },
+    }, {
+        'url': 'https://trovo.live/clip/lc-5285890810184026005',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        vid = self._match_id(url)
+        resp = self._download_json(
+            'https://gql.trovo.live/', vid, data=json.dumps([{
+                'query': '''{
+  batchGetVodDetailInfo(params: {vids: ["%s"]}) {
+    VodDetailInfos
+  }
+}''' % vid,
+            }, {
+                'query': '''{
+  getCommentList(params: {appInfo: {postID: "%s"}, pageSize: 1000000000, preview: {}}) {
+    commentList {
+      author {
+        nickName
+        uid
+      }
+      commentID
+      content
+      createdAt
+      parentID
+    }
+  }
+}''' % vid,
+            }]).encode(), headers={
+                'Content-Type': 'application/json',
+            })
+        vod_detail_info = resp[0]['data']['batchGetVodDetailInfo']['VodDetailInfos'][vid]
+        vod_info = vod_detail_info['vodInfo']
+        title = vod_info['title']
+
+        language = vod_info.get('languageName')
+        formats = []
+        for play_info in (vod_info.get('playInfos') or []):
+            play_url = play_info.get('playUrl')
+            if not play_url:
+                continue
+            format_id = play_info.get('desc')
+            formats.append({
+                'ext': 'mp4',
+                'filesize': int_or_none(play_info.get('fileSize')),
+                'format_id': format_id,
+                'height': int_or_none(format_id[:-1]) if format_id else None,
+                'language': language,
+                'protocol': 'm3u8_native',
+                'tbr': int_or_none(play_info.get('bitrate')),
+                'url': play_url,
+            })
+        self._sort_formats(formats)
+
+        category = vod_info.get('categoryName')
+        get_count = lambda x: int_or_none(vod_info.get(x + 'Num'))
+
+        comment_list = try_get(resp, lambda x: x[1]['data']['getCommentList']['commentList'], list) or []
+        comments = []
+        for comment in comment_list:
+            content = comment.get('content')
+            if not content:
+                continue
+            author = comment.get('author') or {}
+            parent = comment.get('parentID')
+            comments.append({
+                'author': author.get('nickName'),
+                'author_id': str_or_none(author.get('uid')),
+                'id': str_or_none(comment.get('commentID')),
+                'text': content,
+                'timestamp': int_or_none(comment.get('createdAt')),
+                'parent': 'root' if parent == 0 else str_or_none(parent),
+            })
+
+        info = {
+            'id': vid,
+            'title': title,
+            'formats': formats,
+            'thumbnail': vod_info.get('coverUrl'),
+            'timestamp': int_or_none(vod_info.get('publishTs')),
+            'duration': int_or_none(vod_info.get('duration')),
+            'view_count': get_count('watch'),
+            'like_count': get_count('like'),
+            'comment_count': get_count('comment'),
+            'comments': comments,
+            'categories': [category] if category else None,
+        }
+        info.update(self._extract_streamer_info(vod_detail_info))
+        return info

From 657221c81d966115cf6ac263805168e49a48abce Mon Sep 17 00:00:00 2001
From: Brian Marks <bm1549@users.noreply.github.com>
Date: Thu, 21 Jan 2021 10:46:29 -0500
Subject: [PATCH 139/860] [americastestkitchen] Add support for downloading
 entire seasons (#27861)

---
 youtube_dl/extractor/americastestkitchen.py | 67 +++++++++++++++++++++
 youtube_dl/extractor/extractors.py          |  5 +-
 2 files changed, 71 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/americastestkitchen.py b/youtube_dl/extractor/americastestkitchen.py
index 7d2c375c4..35d3220c1 100644
--- a/youtube_dl/extractor/americastestkitchen.py
+++ b/youtube_dl/extractor/americastestkitchen.py
@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import json
 import re
 
 from .common import InfoExtractor
@@ -90,3 +91,69 @@ class AmericasTestKitchenIE(InfoExtractor):
             'series': try_get(episode, lambda x: x['show']['title']),
             'episode': episode.get('title'),
         }
+
+
+class AmericasTestKitchenSeasonIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?(?P<show>americastestkitchen|cookscountry)\.com/episodes/browse/season_(?P<id>\d+)'
+    _TESTS = [{
+        # ATK Season
+        'url': 'https://www.americastestkitchen.com/episodes/browse/season_1',
+        'info_dict': {
+            'id': 'season-1',
+            'title': 'Season 1',
+        },
+        'playlist_count': 13,
+    }, {
+        # Cooks Country Season
+        'url': 'https://www.cookscountry.com/episodes/browse/season_12',
+        'info_dict': {
+            'id': 'season-12',
+            'title': 'Season 12',
+        },
+        'playlist_count': 13,
+    }, {
+        # Multi-digit season
+        'url': 'https://www.americastestkitchen.com/episodes/browse/season_20',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        show_name, season = re.match(self._VALID_URL, url).groups()
+
+        slug = 'atk' if show_name == 'americastestkitchen' else 'cco'
+
+        filters = [
+            'search_season_list:Season %s' % season,
+            'search_document_klass:episode',
+            'search_show_slug:%s' % slug,
+        ]
+
+        season_search = self._download_json(
+            'https://y1fnzxui30-dsn.algolia.net/1/indexes/everest_search_atk_season_desc_production',
+            season, headers={
+                'Origin': 'https://www.%s.com' % show_name,
+                'X-Algolia-API-Key': '8d504d0099ed27c1b73708d22871d805',
+                'X-Algolia-Application-Id': 'Y1FNZXUI30',
+            }, query={
+                'facetFilters': json.dumps(filters),
+                'attributesToRetrieve': 'search_url',
+                'attributesToHighlight': '',
+                # ATK and CCO generally have less than 26 episodes per season
+                'hitsPerPage': '100',
+            })
+
+        entries = [
+            self.url_result(
+                'https://www.%s.com%s' % (show_name, episode['search_url']),
+                'AmericasTestKitchen',
+                try_get(episode, lambda e: e['objectID'].split('_')[-1]))
+            for episode in season_search['hits']
+            if 'search_url' in episode and episode['search_url']
+        ]
+
+        return {
+            '_type': 'playlist',
+            'id': 'season-%s' % season,
+            'title': 'Season %s' % season,
+            'entries': sorted(entries, key=lambda e: e.get('id')),
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 536b184bc..52b8db0f9 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -42,7 +42,10 @@ from .aljazeera import AlJazeeraIE
 from .alphaporno import AlphaPornoIE
 from .amara import AmaraIE
 from .amcnetworks import AMCNetworksIE
-from .americastestkitchen import AmericasTestKitchenIE
+from .americastestkitchen import (
+    AmericasTestKitchenIE,
+    AmericasTestKitchenSeasonIE,
+)
 from .animeondemand import AnimeOnDemandIE
 from .anvato import AnvatoIE
 from .aol import AolIE

From cff72b4cc0330f6d635353083eea7e570036b1ea Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 21 Jan 2021 16:47:55 +0100
Subject: [PATCH 140/860] [americastestkitchen] improve season extraction

---
 youtube_dl/extractor/americastestkitchen.py | 62 ++++++++++-----------
 1 file changed, 31 insertions(+), 31 deletions(-)

diff --git a/youtube_dl/extractor/americastestkitchen.py b/youtube_dl/extractor/americastestkitchen.py
index 35d3220c1..be960c0f9 100644
--- a/youtube_dl/extractor/americastestkitchen.py
+++ b/youtube_dl/extractor/americastestkitchen.py
@@ -99,7 +99,7 @@ class AmericasTestKitchenSeasonIE(InfoExtractor):
         # ATK Season
         'url': 'https://www.americastestkitchen.com/episodes/browse/season_1',
         'info_dict': {
-            'id': 'season-1',
+            'id': 'season_1',
             'title': 'Season 1',
         },
         'playlist_count': 13,
@@ -107,53 +107,53 @@ class AmericasTestKitchenSeasonIE(InfoExtractor):
         # Cooks Country Season
         'url': 'https://www.cookscountry.com/episodes/browse/season_12',
         'info_dict': {
-            'id': 'season-12',
+            'id': 'season_12',
             'title': 'Season 12',
         },
         'playlist_count': 13,
-    }, {
-        # Multi-digit season
-        'url': 'https://www.americastestkitchen.com/episodes/browse/season_20',
-        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        show_name, season = re.match(self._VALID_URL, url).groups()
+        show_name, season_number = re.match(self._VALID_URL, url).groups()
+        season_number = int(season_number)
 
         slug = 'atk' if show_name == 'americastestkitchen' else 'cco'
 
-        filters = [
-            'search_season_list:Season %s' % season,
-            'search_document_klass:episode',
-            'search_show_slug:%s' % slug,
-        ]
+        season = 'Season %d' % season_number
 
         season_search = self._download_json(
-            'https://y1fnzxui30-dsn.algolia.net/1/indexes/everest_search_atk_season_desc_production',
+            'https://y1fnzxui30-dsn.algolia.net/1/indexes/everest_search_%s_season_desc_production' % slug,
             season, headers={
                 'Origin': 'https://www.%s.com' % show_name,
                 'X-Algolia-API-Key': '8d504d0099ed27c1b73708d22871d805',
                 'X-Algolia-Application-Id': 'Y1FNZXUI30',
             }, query={
-                'facetFilters': json.dumps(filters),
-                'attributesToRetrieve': 'search_url',
+                'facetFilters': json.dumps([
+                    'search_season_list:' + season,
+                    'search_document_klass:episode',
+                    'search_show_slug:' + slug,
+                ]),
+                'attributesToRetrieve': 'description,search_%s_episode_number,search_document_date,search_url,title' % slug,
                 'attributesToHighlight': '',
-                # ATK and CCO generally have less than 26 episodes per season
-                'hitsPerPage': '100',
+                'hitsPerPage': 1000,
             })
 
-        entries = [
-            self.url_result(
-                'https://www.%s.com%s' % (show_name, episode['search_url']),
-                'AmericasTestKitchen',
-                try_get(episode, lambda e: e['objectID'].split('_')[-1]))
-            for episode in season_search['hits']
-            if 'search_url' in episode and episode['search_url']
-        ]
+        def entries():
+            for episode in (season_search.get('hits') or []):
+                search_url = episode.get('search_url')
+                if not search_url:
+                    continue
+                yield {
+                    '_type': 'url',
+                    'url': 'https://www.%s.com%s' % (show_name, search_url),
+                    'id': try_get(episode, lambda e: e['objectID'].split('_')[-1]),
+                    'title': episode.get('title'),
+                    'description': episode.get('description'),
+                    'timestamp': unified_timestamp(episode.get('search_document_date')),
+                    'season_number': season_number,
+                    'episode_number': int_or_none(episode.get('search_%s_episode_number' % slug)),
+                    'ie_key': AmericasTestKitchenIE.ie_key(),
+                }
 
-        return {
-            '_type': 'playlist',
-            'id': 'season-%s' % season,
-            'title': 'Season %s' % season,
-            'entries': sorted(entries, key=lambda e: e.get('id')),
-        }
+        return self.playlist_result(
+            entries(), 'season_%d' % season_number, season)

From 8d286bd5b67004d9c5420f6e3f6b7f75d2ba6395 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 21 Jan 2021 17:20:32 +0100
Subject: [PATCH 141/860] [wat] fix format extraction(closes #27901)

---
 youtube_dl/extractor/wat.py | 55 +++++++++----------------------------
 1 file changed, 13 insertions(+), 42 deletions(-)

diff --git a/youtube_dl/extractor/wat.py b/youtube_dl/extractor/wat.py
index 8ef3e0906..7214bfebf 100644
--- a/youtube_dl/extractor/wat.py
+++ b/youtube_dl/extractor/wat.py
@@ -1,12 +1,9 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
-    ExtractorError,
     unified_strdate,
     HEADRequest,
     int_or_none,
@@ -97,46 +94,20 @@ class WatIE(InfoExtractor):
                     return red_url
             return None
 
-        def remove_bitrate_limit(manifest_url):
-            return re.sub(r'(?:max|min)_bitrate=\d+&?', '', manifest_url)
-
         formats = []
-        try:
-            alt_urls = lambda manifest_url: [re.sub(r'(?:wdv|ssm)?\.ism/', repl + '.ism/', manifest_url) for repl in ('', 'ssm')]
-            manifest_urls = self._download_json(
-                'http://www.wat.tv/get/webhtml/' + video_id, video_id)
-            m3u8_url = manifest_urls.get('hls')
-            if m3u8_url:
-                m3u8_url = remove_bitrate_limit(m3u8_url)
-                for m3u8_alt_url in alt_urls(m3u8_url):
-                    formats.extend(self._extract_m3u8_formats(
-                        m3u8_alt_url, video_id, 'mp4',
-                        'm3u8_native', m3u8_id='hls', fatal=False))
-                    formats.extend(self._extract_f4m_formats(
-                        m3u8_alt_url.replace('ios', 'web').replace('.m3u8', '.f4m'),
-                        video_id, f4m_id='hds', fatal=False))
-            mpd_url = manifest_urls.get('mpd')
-            if mpd_url:
-                mpd_url = remove_bitrate_limit(mpd_url)
-                for mpd_alt_url in alt_urls(mpd_url):
-                    formats.extend(self._extract_mpd_formats(
-                        mpd_alt_url, video_id, mpd_id='dash', fatal=False))
-            self._sort_formats(formats)
-        except ExtractorError:
-            abr = 64
-            for vbr, width, height in self._FORMATS:
-                tbr = vbr + abr
-                format_id = 'http-%s' % tbr
-                fmt_url = 'http://dnl.adv.tf1.fr/2/USP-0x0/%s/%s/%s/ssm/%s-%s-64k.mp4' % (video_id[-4:-2], video_id[-2:], video_id, video_id, vbr)
-                if self._is_valid_url(fmt_url, video_id, format_id):
-                    formats.append({
-                        'format_id': format_id,
-                        'url': fmt_url,
-                        'vbr': vbr,
-                        'abr': abr,
-                        'width': width,
-                        'height': height,
-                    })
+        manifest_urls = self._download_json(
+            'http://www.wat.tv/get/webhtml/' + video_id, video_id)
+        m3u8_url = manifest_urls.get('hls')
+        if m3u8_url:
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4',
+                'm3u8_native', m3u8_id='hls', fatal=False))
+        mpd_url = manifest_urls.get('mpd')
+        if mpd_url:
+            formats.extend(self._extract_mpd_formats(
+                mpd_url.replace('://das-q1.tf1.fr/', '://das-q1-ssl.tf1.fr/'),
+                video_id, mpd_id='dash', fatal=False))
+        self._sort_formats(formats)
 
         date_diffusion = first_chapter.get('date_diffusion') or video_data.get('configv4', {}).get('estatS4')
         upload_date = unified_strdate(date_diffusion) if date_diffusion else None

From 3bb7769c405e02fc1078252cafbbd982913fe50c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 21 Jan 2021 17:22:30 +0100
Subject: [PATCH 142/860] [wat] remove unused variable

---
 youtube_dl/extractor/wat.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/youtube_dl/extractor/wat.py b/youtube_dl/extractor/wat.py
index 7214bfebf..f6940b371 100644
--- a/youtube_dl/extractor/wat.py
+++ b/youtube_dl/extractor/wat.py
@@ -43,15 +43,6 @@ class WatIE(InfoExtractor):
         },
     ]
 
-    _FORMATS = (
-        (200, 416, 234),
-        (400, 480, 270),
-        (600, 640, 360),
-        (1200, 640, 360),
-        (1800, 960, 540),
-        (2500, 1280, 720),
-    )
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
         video_id = video_id if video_id.isdigit() and len(video_id) > 6 else compat_str(int(video_id, 36))

From fa8f6d858064cf07b9167b73647545b3007c6b21 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 21 Jan 2021 23:53:09 +0100
Subject: [PATCH 143/860] [comedycentral] fix extraction(closes #27905)

---
 youtube_dl/extractor/comedycentral.py | 141 +++++---------------------
 youtube_dl/extractor/extractors.py    |   3 -
 youtube_dl/extractor/mtv.py           |  23 ++---
 youtube_dl/extractor/spike.py         |  15 +--
 4 files changed, 37 insertions(+), 145 deletions(-)

diff --git a/youtube_dl/extractor/comedycentral.py b/youtube_dl/extractor/comedycentral.py
index d08b909a6..1bfa912be 100644
--- a/youtube_dl/extractor/comedycentral.py
+++ b/youtube_dl/extractor/comedycentral.py
@@ -1,142 +1,51 @@
 from __future__ import unicode_literals
 
 from .mtv import MTVServicesInfoExtractor
-from .common import InfoExtractor
 
 
 class ComedyCentralIE(MTVServicesInfoExtractor):
-    _VALID_URL = r'''(?x)https?://(?:www\.)?cc\.com/
-        (video-clips|episodes|cc-studios|video-collections|shows(?=/[^/]+/(?!full-episodes)))
-        /(?P<title>.*)'''
+    _VALID_URL = r'https?://(?:www\.)?cc\.com/(?:episodes|video(?:-clips)?)/(?P<id>[0-9a-z]{6})'
     _FEED_URL = 'http://comedycentral.com/feeds/mrss/'
 
     _TESTS = [{
-        'url': 'http://www.cc.com/video-clips/kllhuv/stand-up-greg-fitzsimmons--uncensored---too-good-of-a-mother',
-        'md5': 'c4f48e9eda1b16dd10add0744344b6d8',
+        'url': 'http://www.cc.com/video-clips/5ke9v2/the-daily-show-with-trevor-noah-doc-rivers-and-steve-ballmer---the-nba-player-strike',
+        'md5': 'b8acb347177c680ff18a292aa2166f80',
         'info_dict': {
-            'id': 'cef0cbb3-e776-4bc9-b62e-8016deccb354',
+            'id': '89ccc86e-1b02-4f83-b0c9-1d9592ecd025',
             'ext': 'mp4',
-            'title': 'CC:Stand-Up|August 18, 2013|1|0101|Uncensored - Too Good of a Mother',
-            'description': 'After a certain point, breastfeeding becomes c**kblocking.',
-            'timestamp': 1376798400,
-            'upload_date': '20130818',
+            'title': 'The Daily Show with Trevor Noah|August 28, 2020|25|25149|Doc Rivers and Steve Ballmer - The NBA Player Strike',
+            'description': 'md5:5334307c433892b85f4f5e5ac9ef7498',
+            'timestamp': 1598670000,
+            'upload_date': '20200829',
         },
     }, {
-        'url': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/interviews/6yx39d/exclusive-rand-paul-extended-interview',
+        'url': 'http://www.cc.com/episodes/pnzzci/drawn-together--american-idol--parody-clip-show-season-3-ep-314',
         'only_matching': True,
-    }]
-
-
-class ComedyCentralFullEpisodesIE(MTVServicesInfoExtractor):
-    _VALID_URL = r'''(?x)https?://(?:www\.)?cc\.com/
-        (?:full-episodes|shows(?=/[^/]+/full-episodes))
-        /(?P<id>[^?]+)'''
-    _FEED_URL = 'http://comedycentral.com/feeds/mrss/'
-
-    _TESTS = [{
-        'url': 'http://www.cc.com/full-episodes/pv391a/the-daily-show-with-trevor-noah-november-28--2016---ryan-speedo-green-season-22-ep-22028',
-        'info_dict': {
-            'description': 'Donald Trump is accused of exploiting his president-elect status for personal gain, Cuban leader Fidel Castro dies, and Ryan Speedo Green discusses "Sing for Your Life."',
-            'title': 'November 28, 2016 - Ryan Speedo Green',
-        },
-        'playlist_count': 4,
     }, {
-        'url': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-        webpage = self._download_webpage(url, playlist_id)
-        mgid = self._extract_triforce_mgid(webpage, data_zone='t2_lc_promo1')
-        videos_info = self._get_videos_info(mgid)
-        return videos_info
-
-
-class ToshIE(MTVServicesInfoExtractor):
-    IE_DESC = 'Tosh.0'
-    _VALID_URL = r'^https?://tosh\.cc\.com/video-(?:clips|collections)/[^/]+/(?P<videotitle>[^/?#]+)'
-    _FEED_URL = 'http://tosh.cc.com/feeds/mrss'
-
-    _TESTS = [{
-        'url': 'http://tosh.cc.com/video-clips/68g93d/twitter-users-share-summer-plans',
-        'info_dict': {
-            'description': 'Tosh asked fans to share their summer plans.',
-            'title': 'Twitter Users Share Summer Plans',
-        },
-        'playlist': [{
-            'md5': 'f269e88114c1805bb6d7653fecea9e06',
-            'info_dict': {
-                'id': '90498ec2-ed00-11e0-aca6-0026b9414f30',
-                'ext': 'mp4',
-                'title': 'Tosh.0|June 9, 2077|2|211|Twitter Users Share Summer Plans',
-                'description': 'Tosh asked fans to share their summer plans.',
-                'thumbnail': r're:^https?://.*\.jpg',
-                # It's really reported to be published on year 2077
-                'upload_date': '20770610',
-                'timestamp': 3390510600,
-                'subtitles': {
-                    'en': 'mincount:3',
-                },
-            },
-        }]
-    }, {
-        'url': 'http://tosh.cc.com/video-collections/x2iz7k/just-plain-foul/m5q4fp',
+        'url': 'https://www.cc.com/video/k3sdvm/the-daily-show-with-jon-stewart-exclusive-the-fourth-estate',
         'only_matching': True,
     }]
 
 
 class ComedyCentralTVIE(MTVServicesInfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?comedycentral\.tv/(?:staffeln|shows)/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?comedycentral\.tv/folgen/(?P<id>[0-9a-z]{6})'
     _TESTS = [{
-        'url': 'http://www.comedycentral.tv/staffeln/7436-the-mindy-project-staffel-4',
+        'url': 'https://www.comedycentral.tv/folgen/pxdpec/josh-investigates-klimawandel-staffel-1-ep-1',
         'info_dict': {
-            'id': 'local_playlist-f99b626bdfe13568579a',
-            'ext': 'flv',
-            'title': 'Episode_the-mindy-project_shows_season-4_episode-3_full-episode_part1',
+            'id': '15907dc3-ec3c-11e8-a442-0e40cf2fc285',
+            'ext': 'mp4',
+            'title': 'Josh Investigates',
+            'description': 'Steht uns das Ende der Welt bevor?',
         },
-        'params': {
-            # rtmp download
-            'skip_download': True,
-        },
-    }, {
-        'url': 'http://www.comedycentral.tv/shows/1074-workaholics',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.comedycentral.tv/shows/1727-the-mindy-project/bonus',
-        'only_matching': True,
     }]
+    _FEED_URL = 'http://feeds.mtvnservices.com/od/feed/intl-mrss-player-feed'
+    _GEO_COUNTRIES = ['DE']
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        mrss_url = self._search_regex(
-            r'data-mrss=(["\'])(?P<url>(?:(?!\1).)+)\1',
-            webpage, 'mrss url', group='url')
-
-        return self._get_videos_info_from_url(mrss_url, video_id)
-
-
-class ComedyCentralShortnameIE(InfoExtractor):
-    _VALID_URL = r'^:(?P<id>tds|thedailyshow|theopposition)$'
-    _TESTS = [{
-        'url': ':tds',
-        'only_matching': True,
-    }, {
-        'url': ':thedailyshow',
-        'only_matching': True,
-    }, {
-        'url': ':theopposition',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        shortcut_map = {
-            'tds': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes',
-            'thedailyshow': 'http://www.cc.com/shows/the-daily-show-with-trevor-noah/full-episodes',
-            'theopposition': 'http://www.cc.com/shows/the-opposition-with-jordan-klepper/full-episodes',
+    def _get_feed_query(self, uri):
+        return {
+            'accountOverride': 'intl.mtvi.com',
+            'arcEp': 'web.cc.tv',
+            'ep': 'b9032c3a',
+            'imageEp': 'web.cc.tv',
+            'mgid': uri,
         }
-        return self.url_result(shortcut_map[video_id])
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 52b8db0f9..ef57f5556 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -235,11 +235,8 @@ from .cnn import (
 )
 from .coub import CoubIE
 from .comedycentral import (
-    ComedyCentralFullEpisodesIE,
     ComedyCentralIE,
-    ComedyCentralShortnameIE,
     ComedyCentralTVIE,
-    ToshIE,
 )
 from .commonmistakes import CommonMistakesIE, UnicodeBOMIE
 from .commonprotocols import (
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index df1034fc5..f5e30d22d 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -253,6 +253,10 @@ class MTVServicesInfoExtractor(InfoExtractor):
 
         return try_get(feed, lambda x: x['result']['data']['id'], compat_str)
 
+    @staticmethod
+    def _extract_child_with_type(parent, t):
+        return next(c for c in parent['children'] if c.get('type') == t)
+
     def _extract_mgid(self, webpage):
         try:
             # the url can be http://media.mtvnservices.com/fb/{mgid}.swf
@@ -278,6 +282,13 @@ class MTVServicesInfoExtractor(InfoExtractor):
         if not mgid:
             mgid = self._extract_triforce_mgid(webpage)
 
+        if not mgid:
+            data = self._parse_json(self._search_regex(
+                r'__DATA__\s*=\s*({.+?});', webpage, 'data'), None)
+            main_container = self._extract_child_with_type(data, 'MainContainer')
+            video_player = self._extract_child_with_type(main_container, 'VideoPlayer')
+            mgid = video_player['props']['media']['video']['config']['uri']
+
         return mgid
 
     def _real_extract(self, url):
@@ -349,18 +360,6 @@ class MTVIE(MTVServicesInfoExtractor):
         'only_matching': True,
     }]
 
-    @staticmethod
-    def extract_child_with_type(parent, t):
-        children = parent['children']
-        return next(c for c in children if c.get('type') == t)
-
-    def _extract_mgid(self, webpage):
-        data = self._parse_json(self._search_regex(
-            r'__DATA__\s*=\s*({.+?});', webpage, 'data'), None)
-        main_container = self.extract_child_with_type(data, 'MainContainer')
-        video_player = self.extract_child_with_type(main_container, 'VideoPlayer')
-        return video_player['props']['media']['video']['config']['uri']
-
 
 class MTVJapanIE(MTVServicesInfoExtractor):
     IE_NAME = 'mtvjapan'
diff --git a/youtube_dl/extractor/spike.py b/youtube_dl/extractor/spike.py
index 4c5e3f7c2..5805f3d44 100644
--- a/youtube_dl/extractor/spike.py
+++ b/youtube_dl/extractor/spike.py
@@ -20,9 +20,6 @@ class BellatorIE(MTVServicesInfoExtractor):
     _FEED_URL = 'http://www.bellator.com/feeds/mrss/'
     _GEO_COUNTRIES = ['US']
 
-    def _extract_mgid(self, webpage):
-        return self._extract_triforce_mgid(webpage)
-
 
 class ParamountNetworkIE(MTVServicesInfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?paramountnetwork\.com/[^/]+/[\da-z]{6}(?:[/?#&]|$)'
@@ -46,16 +43,6 @@ class ParamountNetworkIE(MTVServicesInfoExtractor):
     def _get_feed_query(self, uri):
         return {
             'arcEp': 'paramountnetwork.com',
+            'imageEp': 'paramountnetwork.com',
             'mgid': uri,
         }
-
-    def _extract_mgid(self, webpage):
-        root_data = self._parse_json(self._search_regex(
-            r'window\.__DATA__\s*=\s*({.+})',
-            webpage, 'data'), None)
-
-        def find_sub_data(data, data_type):
-            return next(c for c in data['children'] if c.get('type') == data_type)
-
-        c = find_sub_data(find_sub_data(root_data, 'MainContainer'), 'VideoPlayer')
-        return c['props']['media']['video']['config']['uri']

From 4542e3e55538a8e7115bde777e7ddbd781c2e446 Mon Sep 17 00:00:00 2001
From: aarubui <aarubui@users.noreply.github.com>
Date: Fri, 22 Jan 2021 19:56:51 +1100
Subject: [PATCH 144/860] [njpwworld] fix extraction (#27890)

---
 youtube_dl/extractor/njpwworld.py | 54 ++++++++++++++++---------------
 1 file changed, 28 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/njpwworld.py b/youtube_dl/extractor/njpwworld.py
index 025c5d249..3639d142f 100644
--- a/youtube_dl/extractor/njpwworld.py
+++ b/youtube_dl/extractor/njpwworld.py
@@ -6,30 +6,40 @@ import re
 from .common import InfoExtractor
 from ..compat import compat_urlparse
 from ..utils import (
-    extract_attributes,
     get_element_by_class,
     urlencode_postdata,
 )
 
 
 class NJPWWorldIE(InfoExtractor):
-    _VALID_URL = r'https?://njpwworld\.com/p/(?P<id>[a-z0-9_]+)'
+    _VALID_URL = r'https?://(front\.)?njpwworld\.com/p/(?P<id>[a-z0-9_]+)'
     IE_DESC = '新日本プロレスワールド'
     _NETRC_MACHINE = 'njpwworld'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://njpwworld.com/p/s_series_00155_1_9/',
         'info_dict': {
             'id': 's_series_00155_1_9',
             'ext': 'mp4',
-            'title': '第9試合　ランディ・サベージ　vs　リック・スタイナー',
+            'title': '闘強導夢2000 2000年1月4日 東京ドーム 第9試合 ランディ・サベージ VS リック・スタイナー',
             'tags': list,
         },
         'params': {
             'skip_download': True,  # AES-encrypted m3u8
         },
         'skip': 'Requires login',
-    }
+    }, {
+        'url': 'https://front.njpwworld.com/p/s_series_00563_16_bs',
+        'info_dict': {
+            'id': 's_series_00563_16_bs',
+            'ext': 'mp4',
+            'title': 'WORLD TAG LEAGUE 2020 & BEST OF THE SUPER Jr.27 2020年12月6日 福岡・福岡国際センター バックステージコメント（字幕あり）',
+            'tags': ["福岡・福岡国際センター", "バックステージコメント", "2020", "20年代"],
+        },
+        'params': {
+            'skip_download': True,
+        },
+    }]
 
     _LOGIN_URL = 'https://front.njpwworld.com/auth/login'
 
@@ -64,35 +74,27 @@ class NJPWWorldIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         formats = []
-        for mobj in re.finditer(r'<a[^>]+\bhref=(["\'])/player.+?[^>]*>', webpage):
-            player = extract_attributes(mobj.group(0))
-            player_path = player.get('href')
-            if not player_path:
-                continue
-            kind = self._search_regex(
-                r'(low|high)$', player.get('class') or '', 'kind',
-                default='low')
+        for kind, vid in re.findall(r'if\s+\(\s*imageQualityType\s*==\s*\'([^\']+)\'\s*\)\s*{\s*video_id\s*=\s*"(\d+)"', webpage):
+            player_path = '/intent?id=%s&type=url' % vid
             player_url = compat_urlparse.urljoin(url, player_path)
-            player_page = self._download_webpage(
-                player_url, video_id, note='Downloading player page')
-            entries = self._parse_html5_media_entries(
-                player_url, player_page, video_id, m3u8_id='hls-%s' % kind,
-                m3u8_entry_protocol='m3u8_native')
-            kind_formats = entries[0]['formats']
-            for f in kind_formats:
-                f['quality'] = 2 if kind == 'high' else 1
-            formats.extend(kind_formats)
+            formats.append({
+                'url': player_url,
+                'format_id': kind,
+                'ext': 'mp4',
+                'protocol': 'm3u8',
+                'quality': 2 if kind == 'high' else 1,
+            })
 
         self._sort_formats(formats)
 
-        post_content = get_element_by_class('post-content', webpage)
+        tag_block = get_element_by_class('tag-block', webpage)
         tags = re.findall(
-            r'<li[^>]+class="tag-[^"]+"><a[^>]*>([^<]+)</a></li>', post_content
-        ) if post_content else None
+            r'<a[^>]+class="tag-[^"]+"[^>]*>([^<]+)</a>', tag_block
+        ) if tag_block else None
 
         return {
             'id': video_id,
-            'title': self._og_search_title(webpage),
+            'title': get_element_by_class('article-title', webpage) or self._og_search_title(webpage),
             'formats': formats,
             'tags': tags,
         }

From 142c584063ec02406e636522fe11d0d2be22b299 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 22 Jan 2021 23:51:22 +0700
Subject: [PATCH 145/860] Introduce --output-na-placeholder (closes #27896)

---
 README.md               |  2 +-
 test/test_YoutubeDL.py  | 15 +++++++++++----
 youtube_dl/YoutubeDL.py |  7 ++++---
 youtube_dl/__init__.py  |  1 +
 youtube_dl/options.py   |  4 ++++
 5 files changed, 21 insertions(+), 8 deletions(-)

diff --git a/README.md b/README.md
index 85fed6d3a..8c2569aaa 100644
--- a/README.md
+++ b/README.md
@@ -583,7 +583,7 @@ Available for the media that is a track or a part of a music album:
  - `disc_number` (numeric): Number of the disc or other physical medium the track belongs to
  - `release_year` (numeric): Year (YYYY) when the album was released
 
-Each aforementioned sequence when referenced in an output template will be replaced by the actual value corresponding to the sequence name. Note that some of the sequences are not guaranteed to be present since they depend on the metadata obtained by a particular extractor. Such sequences will be replaced with `NA`.
+Each aforementioned sequence when referenced in an output template will be replaced by the actual value corresponding to the sequence name. Note that some of the sequences are not guaranteed to be present since they depend on the metadata obtained by a particular extractor. Such sequences will be replaced with placeholder value provided with `--output-na-placeholder` (`NA` by default).
 
 For example for `-o %(title)s-%(id)s.%(ext)s` and an mp4 video with title `youtube-dl test video` and id `BaW_jenozKcj`, this will result in a `youtube-dl test video-BaW_jenozKcj.mp4` file created in the current directory.
 
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 4d62ba145..a35effe0e 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -633,13 +633,20 @@ class TestYoutubeDL(unittest.TestCase):
             'title2': '%PATH%',
         }
 
-        def fname(templ):
-            ydl = YoutubeDL({'outtmpl': templ})
+        def fname(templ, na_placeholder='NA'):
+            params = {'outtmpl': templ}
+            if na_placeholder != 'NA':
+                params['outtmpl_na_placeholder'] = na_placeholder
+            ydl = YoutubeDL(params)
             return ydl.prepare_filename(info)
         self.assertEqual(fname('%(id)s.%(ext)s'), '1234.mp4')
         self.assertEqual(fname('%(id)s-%(width)s.%(ext)s'), '1234-NA.mp4')
-        # Replace missing fields with 'NA'
-        self.assertEqual(fname('%(uploader_date)s-%(id)s.%(ext)s'), 'NA-1234.mp4')
+        NA_TEST_OUTTMPL = '%(uploader_date)s-%(width)d-%(id)s.%(ext)s'
+        # Replace missing fields with 'NA' by default
+        self.assertEqual(fname(NA_TEST_OUTTMPL), 'NA-NA-1234.mp4')
+        # Or by provided placeholder
+        self.assertEqual(fname(NA_TEST_OUTTMPL, na_placeholder='none'), 'none-none-1234.mp4')
+        self.assertEqual(fname(NA_TEST_OUTTMPL, na_placeholder=''), '--1234.mp4')
         self.assertEqual(fname('%(height)d.%(ext)s'), '1080.mp4')
         self.assertEqual(fname('%(height)6d.%(ext)s'), '  1080.mp4')
         self.assertEqual(fname('%(height)-6d.%(ext)s'), '1080  .mp4')
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index efd42fa63..ecac31f7a 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -163,6 +163,7 @@ class YoutubeDL(object):
     simulate:          Do not download the video files.
     format:            Video format code. See options.py for more information.
     outtmpl:           Template for output names.
+    outtmpl_na_placeholder: Placeholder for unavailable meta fields.
     restrictfilenames: Do not allow "&" and spaces in file names
     ignoreerrors:      Do not stop on download errors.
     force_generic_extractor: Force downloader to use the generic extractor
@@ -658,7 +659,7 @@ class YoutubeDL(object):
             template_dict = dict((k, v if isinstance(v, compat_numeric_types) else sanitize(k, v))
                                  for k, v in template_dict.items()
                                  if v is not None and not isinstance(v, (list, tuple, dict)))
-            template_dict = collections.defaultdict(lambda: 'NA', template_dict)
+            template_dict = collections.defaultdict(lambda: self.params.get('outtmpl_na_placeholder', 'NA'), template_dict)
 
             outtmpl = self.params.get('outtmpl', DEFAULT_OUTTMPL)
 
@@ -678,8 +679,8 @@ class YoutubeDL(object):
 
             # Missing numeric fields used together with integer presentation types
             # in format specification will break the argument substitution since
-            # string 'NA' is returned for missing fields. We will patch output
-            # template for missing fields to meet string presentation type.
+            # string NA placeholder is returned for missing fields. We will patch
+            # output template for missing fields to meet string presentation type.
             for numeric_field in self._NUMERIC_FIELDS:
                 if numeric_field not in template_dict:
                     # As of [1] format syntax is:
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 9a659fc65..e1bd67919 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -340,6 +340,7 @@ def _real_main(argv=None):
         'format': opts.format,
         'listformats': opts.listformats,
         'outtmpl': outtmpl,
+        'outtmpl_na_placeholder': opts.outtmpl_na_placeholder,
         'autonumber_size': opts.autonumber_size,
         'autonumber_start': opts.autonumber_start,
         'restrictfilenames': opts.restrictfilenames,
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 3000ba41e..6b0c62e19 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -689,6 +689,10 @@ def parseOpts(overrideArguments=None):
         '-o', '--output',
         dest='outtmpl', metavar='TEMPLATE',
         help=('Output filename template, see the "OUTPUT TEMPLATE" for all the info'))
+    filesystem.add_option(
+        '--output-na-placeholder',
+        dest='outtmpl_na_placeholder', metavar='PLACEHOLDER', default='NA',
+        help=('Placeholder value for unavailable meta fields in output filename template (default is "%default")'))
     filesystem.add_option(
         '--autonumber-size',
         dest='autonumber_size', metavar='NUMBER', type=int,

From 5519bba3e14d05ccc8c7114e2f2909294c65a26e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2021 00:00:53 +0700
Subject: [PATCH 146/860] [options] Clarify --extract-audio help string (closes
 #27878)

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 6b0c62e19..241cf110f 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -786,7 +786,7 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '-x', '--extract-audio',
         action='store_true', dest='extractaudio', default=False,
-        help='Convert video files to audio-only files (requires ffmpeg or avconv and ffprobe or avprobe)')
+        help='Convert video files to audio-only files (requires ffmpeg/avconv and ffprobe/avprobe)')
     postproc.add_option(
         '--audio-format', metavar='FORMAT', dest='audioformat', default='best',
         help='Specify audio format: "best", "aac", "flac", "mp3", "m4a", "opus", "vorbis", or "wav"; "%default" by default; No effect without -x')

From d8dab85419ea7e35bd234479abe751334f3e7116 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Aur=C3=A9lien=20Grosdidier?= <aurelien.grosdidier@gmail.com>
Date: Fri, 22 Jan 2021 19:03:45 +0100
Subject: [PATCH 147/860] [franceculture] Fix extraction (closes #27891)
 (#27903)

Co-authored-by: Sergey M. <dstftw@gmail.com>
---
 youtube_dl/extractor/franceculture.py | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/franceculture.py b/youtube_dl/extractor/franceculture.py
index 306b45fc9..7e9ceabbc 100644
--- a/youtube_dl/extractor/franceculture.py
+++ b/youtube_dl/extractor/franceculture.py
@@ -20,7 +20,7 @@ class FranceCultureIE(InfoExtractor):
             'title': 'Rendez-vous au pays des geeks',
             'thumbnail': r're:^https?://.*\.jpg$',
             'upload_date': '20140301',
-            'timestamp': 1393642916,
+            'timestamp': 1393700400,
             'vcodec': 'none',
         }
     }
@@ -36,12 +36,12 @@ class FranceCultureIE(InfoExtractor):
                     </h1>|
                     <div[^>]+class="[^"]*?(?:title-zone-diffusion|heading-zone-(?:wrapper|player-button))[^"]*?"[^>]*>
                 ).*?
-                (<button[^>]+data-asset-source="[^"]+"[^>]+>)
+                (<button[^>]+data-(?:url|asset-source)="[^"]+"[^>]+>)
             ''',
             webpage, 'video data'))
 
-        video_url = video_data['data-asset-source']
-        title = video_data.get('data-asset-title') or self._og_search_title(webpage)
+        video_url = video_data.get('data-url') or video_data['data-asset-source']
+        title = video_data.get('data-asset-title') or video_data.get('data-diffusion-title') or self._og_search_title(webpage)
 
         description = self._html_search_regex(
             r'(?s)<div[^>]+class="intro"[^>]*>.*?<h2>(.+?)</h2>',
@@ -64,6 +64,6 @@ class FranceCultureIE(InfoExtractor):
             'ext': ext,
             'vcodec': 'none' if ext == 'mp3' else None,
             'uploader': uploader,
-            'timestamp': int_or_none(video_data.get('data-asset-created-date')),
+            'timestamp': int_or_none(video_data.get('data-start-time')) or int_or_none(video_data.get('data-asset-created-date')),
             'duration': int_or_none(video_data.get('data-duration')),
         }

From f08c31cf33beb0687c4df58b83e319ec8dfe03ad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 23 Jan 2021 01:10:35 +0700
Subject: [PATCH 148/860] [franceculture] Make thumbnail optional (closes
 #18807)

---
 youtube_dl/extractor/franceculture.py | 10 +++++++---
 1 file changed, 7 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/franceculture.py b/youtube_dl/extractor/franceculture.py
index 7e9ceabbc..14f4cb489 100644
--- a/youtube_dl/extractor/franceculture.py
+++ b/youtube_dl/extractor/franceculture.py
@@ -11,7 +11,7 @@ from ..utils import (
 
 class FranceCultureIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?franceculture\.fr/emissions/(?:[^/]+/)*(?P<id>[^/?#&]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.franceculture.fr/emissions/carnet-nomade/rendez-vous-au-pays-des-geeks',
         'info_dict': {
             'id': 'rendez-vous-au-pays-des-geeks',
@@ -23,7 +23,11 @@ class FranceCultureIE(InfoExtractor):
             'timestamp': 1393700400,
             'vcodec': 'none',
         }
-    }
+    }, {
+        # no thumbnail
+        'url': 'https://www.franceculture.fr/emissions/la-recherche-montre-en-main/la-recherche-montre-en-main-du-mercredi-10-octobre-2018',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         display_id = self._match_id(url)
@@ -48,7 +52,7 @@ class FranceCultureIE(InfoExtractor):
             webpage, 'description', default=None)
         thumbnail = self._search_regex(
             r'(?s)<figure[^>]+itemtype="https://schema.org/ImageObject"[^>]*>.*?<img[^>]+(?:data-dejavu-)?src="([^"]+)"',
-            webpage, 'thumbnail', fatal=False)
+            webpage, 'thumbnail', default=None)
         uploader = self._html_search_regex(
             r'(?s)<span class="author">(.*?)</span>',
             webpage, 'uploader', default=None)

From dbf3fa8af67dfdab42d56fdc5f35610658bc2746 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Jan 2021 17:36:40 +0700
Subject: [PATCH 149/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 27f01c438..f09049fea 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,27 @@
+version <unreleased>
+
+Core
+* Introduce --output-na-placeholder (#27896)
+
+Extractors
+* [franceculture] Make thumbnail optional (#18807)
+* [franceculture] Fix extraction (#27891, #27903)
+* [njpwworld] Fix extraction (#27890)
+* [comedycentral] Fix extraction (#27905)
+* [wat] Fix format extraction (#27901)
++ [americastestkitchen:season] Add support for seasons (#27861)
++ [trovo] Add support for trovo.live (#26125)
++ [aol] Add support for yahoo videos (#26650)
+* [yahoo] Fix single video extraction
+* [lbry] Unescape lbry URI (#27872)
+* [9gag] Fix and improve extraction (#23022)
+* [americastestkitchen] Improve metadata extraction for ATK episodes (#27860)
+* [aljazeera] Fix extraction (#20911, #27779)
++ [minds] Add support for minds.com (#17934)
+* [ard] Fix title and description extraction (#27761)
++ [spotify] Add support for Spotify Podcasts (#27443)
+
+
 version 2021.01.16
 
 Core

From 186cbaffb954f00c4ff1f58f9fe378c65b4ef87b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Jan 2021 18:00:58 +0700
Subject: [PATCH 150/860] release 2021.01.24

---
 .github/ISSUE_TEMPLATE/1_broken_site.md       |   6 +-
 .../ISSUE_TEMPLATE/2_site_support_request.md  |   4 +-
 .../ISSUE_TEMPLATE/3_site_feature_request.md  |   4 +-
 .github/ISSUE_TEMPLATE/4_bug_report.md        |   6 +-
 .github/ISSUE_TEMPLATE/5_feature_request.md   |   4 +-
 ChangeLog                                     |   2 +-
 README.md                                     | 761 +++++++++---------
 docs/supportedsites.md                        |  13 +-
 youtube_dl/version.py                         |   2 +-
 9 files changed, 422 insertions(+), 380 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index aedcfa6b3..935806b5e 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.01.16**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.24**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.01.16
+ [debug] youtube-dl version 2021.01.24
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index 5c0dfea4e..453a5e147 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.01.16**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.24**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 772147a75..07094c10d 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.01.16**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.24**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index 2fcaa3a23..b234f8ccd 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.01.16**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.24**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.01.16
+ [debug] youtube-dl version 2021.01.24
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index f1adfce8f..557e59ca5 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.01.16**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.24**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index f09049fea..9b2f38b25 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.01.24
 
 Core
 * Introduce --output-na-placeholder (#27896)
diff --git a/README.md b/README.md
index 8c2569aaa..94c34d89a 100644
--- a/README.md
+++ b/README.md
@@ -52,394 +52,431 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     youtube-dl [OPTIONS] URL [URL...]
 
 # OPTIONS
-    -h, --help                       Print this help text and exit
-    --version                        Print program version and exit
-    -U, --update                     Update this program to latest version. Make
-                                     sure that you have sufficient permissions
-                                     (run with sudo if needed)
-    -i, --ignore-errors              Continue on download errors, for example to
-                                     skip unavailable videos in a playlist
-    --abort-on-error                 Abort downloading of further videos (in the
-                                     playlist or the command line) if an error
-                                     occurs
-    --dump-user-agent                Display the current browser identification
-    --list-extractors                List all supported extractors
-    --extractor-descriptions         Output descriptions of all supported
-                                     extractors
-    --force-generic-extractor        Force extraction to use the generic
-                                     extractor
-    --default-search PREFIX          Use this prefix for unqualified URLs. For
-                                     example "gvsearch2:" downloads two videos
-                                     from google videos for youtube-dl "large
-                                     apple". Use the value "auto" to let
-                                     youtube-dl guess ("auto_warning" to emit a
-                                     warning when guessing). "error" just throws
-                                     an error. The default value "fixup_error"
-                                     repairs broken URLs, but emits an error if
-                                     this is not possible instead of searching.
-    --ignore-config                  Do not read configuration files. When given
-                                     in the global configuration file
-                                     /etc/youtube-dl.conf: Do not read the user
-                                     configuration in ~/.config/youtube-
-                                     dl/config (%APPDATA%/youtube-dl/config.txt
-                                     on Windows)
-    --config-location PATH           Location of the configuration file; either
-                                     the path to the config or its containing
-                                     directory.
-    --flat-playlist                  Do not extract the videos of a playlist,
-                                     only list them.
-    --mark-watched                   Mark videos watched (YouTube only)
-    --no-mark-watched                Do not mark videos watched (YouTube only)
-    --no-color                       Do not emit color codes in output
+    -h, --help                           Print this help text and exit
+    --version                            Print program version and exit
+    -U, --update                         Update this program to latest version.
+                                         Make sure that you have sufficient
+                                         permissions (run with sudo if needed)
+    -i, --ignore-errors                  Continue on download errors, for
+                                         example to skip unavailable videos in a
+                                         playlist
+    --abort-on-error                     Abort downloading of further videos (in
+                                         the playlist or the command line) if an
+                                         error occurs
+    --dump-user-agent                    Display the current browser
+                                         identification
+    --list-extractors                    List all supported extractors
+    --extractor-descriptions             Output descriptions of all supported
+                                         extractors
+    --force-generic-extractor            Force extraction to use the generic
+                                         extractor
+    --default-search PREFIX              Use this prefix for unqualified URLs.
+                                         For example "gvsearch2:" downloads two
+                                         videos from google videos for youtube-
+                                         dl "large apple". Use the value "auto"
+                                         to let youtube-dl guess ("auto_warning"
+                                         to emit a warning when guessing).
+                                         "error" just throws an error. The
+                                         default value "fixup_error" repairs
+                                         broken URLs, but emits an error if this
+                                         is not possible instead of searching.
+    --ignore-config                      Do not read configuration files. When
+                                         given in the global configuration file
+                                         /etc/youtube-dl.conf: Do not read the
+                                         user configuration in
+                                         ~/.config/youtube-dl/config
+                                         (%APPDATA%/youtube-dl/config.txt on
+                                         Windows)
+    --config-location PATH               Location of the configuration file;
+                                         either the path to the config or its
+                                         containing directory.
+    --flat-playlist                      Do not extract the videos of a
+                                         playlist, only list them.
+    --mark-watched                       Mark videos watched (YouTube only)
+    --no-mark-watched                    Do not mark videos watched (YouTube
+                                         only)
+    --no-color                           Do not emit color codes in output
 
 ## Network Options:
-    --proxy URL                      Use the specified HTTP/HTTPS/SOCKS proxy.
-                                     To enable SOCKS proxy, specify a proper
-                                     scheme. For example
-                                     socks5://127.0.0.1:1080/. Pass in an empty
-                                     string (--proxy "") for direct connection
-    --socket-timeout SECONDS         Time to wait before giving up, in seconds
-    --source-address IP              Client-side IP address to bind to
-    -4, --force-ipv4                 Make all connections via IPv4
-    -6, --force-ipv6                 Make all connections via IPv6
+    --proxy URL                          Use the specified HTTP/HTTPS/SOCKS
+                                         proxy. To enable SOCKS proxy, specify a
+                                         proper scheme. For example
+                                         socks5://127.0.0.1:1080/. Pass in an
+                                         empty string (--proxy "") for direct
+                                         connection
+    --socket-timeout SECONDS             Time to wait before giving up, in
+                                         seconds
+    --source-address IP                  Client-side IP address to bind to
+    -4, --force-ipv4                     Make all connections via IPv4
+    -6, --force-ipv6                     Make all connections via IPv6
 
 ## Geo Restriction:
-    --geo-verification-proxy URL     Use this proxy to verify the IP address for
-                                     some geo-restricted sites. The default
-                                     proxy specified by --proxy (or none, if the
-                                     option is not present) is used for the
-                                     actual downloading.
-    --geo-bypass                     Bypass geographic restriction via faking
-                                     X-Forwarded-For HTTP header
-    --no-geo-bypass                  Do not bypass geographic restriction via
-                                     faking X-Forwarded-For HTTP header
-    --geo-bypass-country CODE        Force bypass geographic restriction with
-                                     explicitly provided two-letter ISO 3166-2
-                                     country code
-    --geo-bypass-ip-block IP_BLOCK   Force bypass geographic restriction with
-                                     explicitly provided IP block in CIDR
-                                     notation
+    --geo-verification-proxy URL         Use this proxy to verify the IP address
+                                         for some geo-restricted sites. The
+                                         default proxy specified by --proxy (or
+                                         none, if the option is not present) is
+                                         used for the actual downloading.
+    --geo-bypass                         Bypass geographic restriction via
+                                         faking X-Forwarded-For HTTP header
+    --no-geo-bypass                      Do not bypass geographic restriction
+                                         via faking X-Forwarded-For HTTP header
+    --geo-bypass-country CODE            Force bypass geographic restriction
+                                         with explicitly provided two-letter ISO
+                                         3166-2 country code
+    --geo-bypass-ip-block IP_BLOCK       Force bypass geographic restriction
+                                         with explicitly provided IP block in
+                                         CIDR notation
 
 ## Video Selection:
-    --playlist-start NUMBER          Playlist video to start at (default is 1)
-    --playlist-end NUMBER            Playlist video to end at (default is last)
-    --playlist-items ITEM_SPEC       Playlist video items to download. Specify
-                                     indices of the videos in the playlist
-                                     separated by commas like: "--playlist-items
-                                     1,2,5,8" if you want to download videos
-                                     indexed 1, 2, 5, 8 in the playlist. You can
-                                     specify range: "--playlist-items
-                                     1-3,7,10-13", it will download the videos
-                                     at index 1, 2, 3, 7, 10, 11, 12 and 13.
-    --match-title REGEX              Download only matching titles (regex or
-                                     caseless sub-string)
-    --reject-title REGEX             Skip download for matching titles (regex or
-                                     caseless sub-string)
-    --max-downloads NUMBER           Abort after downloading NUMBER files
-    --min-filesize SIZE              Do not download any videos smaller than
-                                     SIZE (e.g. 50k or 44.6m)
-    --max-filesize SIZE              Do not download any videos larger than SIZE
-                                     (e.g. 50k or 44.6m)
-    --date DATE                      Download only videos uploaded in this date
-    --datebefore DATE                Download only videos uploaded on or before
-                                     this date (i.e. inclusive)
-    --dateafter DATE                 Download only videos uploaded on or after
-                                     this date (i.e. inclusive)
-    --min-views COUNT                Do not download any videos with less than
-                                     COUNT views
-    --max-views COUNT                Do not download any videos with more than
-                                     COUNT views
-    --match-filter FILTER            Generic video filter. Specify any key (see
-                                     the "OUTPUT TEMPLATE" for a list of
-                                     available keys) to match if the key is
-                                     present, !key to check if the key is not
-                                     present, key > NUMBER (like "comment_count
-                                     > 12", also works with >=, <, <=, !=, =) to
-                                     compare against a number, key = 'LITERAL'
-                                     (like "uploader = 'Mike Smith'", also works
-                                     with !=) to match against a string literal
-                                     and & to require multiple matches. Values
-                                     which are not known are excluded unless you
-                                     put a question mark (?) after the operator.
-                                     For example, to only match videos that have
-                                     been liked more than 100 times and disliked
-                                     less than 50 times (or the dislike
-                                     functionality is not available at the given
-                                     service), but who also have a description,
-                                     use --match-filter "like_count > 100 &
-                                     dislike_count <? 50 & description" .
-    --no-playlist                    Download only the video, if the URL refers
-                                     to a video and a playlist.
-    --yes-playlist                   Download the playlist, if the URL refers to
-                                     a video and a playlist.
-    --age-limit YEARS                Download only videos suitable for the given
-                                     age
-    --download-archive FILE          Download only videos not listed in the
-                                     archive file. Record the IDs of all
-                                     downloaded videos in it.
-    --include-ads                    Download advertisements as well
-                                     (experimental)
+    --playlist-start NUMBER              Playlist video to start at (default is
+                                         1)
+    --playlist-end NUMBER                Playlist video to end at (default is
+                                         last)
+    --playlist-items ITEM_SPEC           Playlist video items to download.
+                                         Specify indices of the videos in the
+                                         playlist separated by commas like: "--
+                                         playlist-items 1,2,5,8" if you want to
+                                         download videos indexed 1, 2, 5, 8 in
+                                         the playlist. You can specify range: "
+                                         --playlist-items 1-3,7,10-13", it will
+                                         download the videos at index 1, 2, 3,
+                                         7, 10, 11, 12 and 13.
+    --match-title REGEX                  Download only matching titles (regex or
+                                         caseless sub-string)
+    --reject-title REGEX                 Skip download for matching titles
+                                         (regex or caseless sub-string)
+    --max-downloads NUMBER               Abort after downloading NUMBER files
+    --min-filesize SIZE                  Do not download any videos smaller than
+                                         SIZE (e.g. 50k or 44.6m)
+    --max-filesize SIZE                  Do not download any videos larger than
+                                         SIZE (e.g. 50k or 44.6m)
+    --date DATE                          Download only videos uploaded in this
+                                         date
+    --datebefore DATE                    Download only videos uploaded on or
+                                         before this date (i.e. inclusive)
+    --dateafter DATE                     Download only videos uploaded on or
+                                         after this date (i.e. inclusive)
+    --min-views COUNT                    Do not download any videos with less
+                                         than COUNT views
+    --max-views COUNT                    Do not download any videos with more
+                                         than COUNT views
+    --match-filter FILTER                Generic video filter. Specify any key
+                                         (see the "OUTPUT TEMPLATE" for a list
+                                         of available keys) to match if the key
+                                         is present, !key to check if the key is
+                                         not present, key > NUMBER (like
+                                         "comment_count > 12", also works with
+                                         >=, <, <=, !=, =) to compare against a
+                                         number, key = 'LITERAL' (like "uploader
+                                         = 'Mike Smith'", also works with !=) to
+                                         match against a string literal and & to
+                                         require multiple matches. Values which
+                                         are not known are excluded unless you
+                                         put a question mark (?) after the
+                                         operator. For example, to only match
+                                         videos that have been liked more than
+                                         100 times and disliked less than 50
+                                         times (or the dislike functionality is
+                                         not available at the given service),
+                                         but who also have a description, use
+                                         --match-filter "like_count > 100 &
+                                         dislike_count <? 50 & description" .
+    --no-playlist                        Download only the video, if the URL
+                                         refers to a video and a playlist.
+    --yes-playlist                       Download the playlist, if the URL
+                                         refers to a video and a playlist.
+    --age-limit YEARS                    Download only videos suitable for the
+                                         given age
+    --download-archive FILE              Download only videos not listed in the
+                                         archive file. Record the IDs of all
+                                         downloaded videos in it.
+    --include-ads                        Download advertisements as well
+                                         (experimental)
 
 ## Download Options:
-    -r, --limit-rate RATE            Maximum download rate in bytes per second
-                                     (e.g. 50K or 4.2M)
-    -R, --retries RETRIES            Number of retries (default is 10), or
-                                     "infinite".
-    --fragment-retries RETRIES       Number of retries for a fragment (default
-                                     is 10), or "infinite" (DASH, hlsnative and
-                                     ISM)
-    --skip-unavailable-fragments     Skip unavailable fragments (DASH, hlsnative
-                                     and ISM)
-    --abort-on-unavailable-fragment  Abort downloading when some fragment is not
-                                     available
-    --keep-fragments                 Keep downloaded fragments on disk after
-                                     downloading is finished; fragments are
-                                     erased by default
-    --buffer-size SIZE               Size of download buffer (e.g. 1024 or 16K)
-                                     (default is 1024)
-    --no-resize-buffer               Do not automatically adjust the buffer
-                                     size. By default, the buffer size is
-                                     automatically resized from an initial value
-                                     of SIZE.
-    --http-chunk-size SIZE           Size of a chunk for chunk-based HTTP
-                                     downloading (e.g. 10485760 or 10M) (default
-                                     is disabled). May be useful for bypassing
-                                     bandwidth throttling imposed by a webserver
-                                     (experimental)
-    --playlist-reverse               Download playlist videos in reverse order
-    --playlist-random                Download playlist videos in random order
-    --xattr-set-filesize             Set file xattribute ytdl.filesize with
-                                     expected file size
-    --hls-prefer-native              Use the native HLS downloader instead of
-                                     ffmpeg
-    --hls-prefer-ffmpeg              Use ffmpeg instead of the native HLS
-                                     downloader
-    --hls-use-mpegts                 Use the mpegts container for HLS videos,
-                                     allowing to play the video while
-                                     downloading (some players may not be able
-                                     to play it)
-    --external-downloader COMMAND    Use the specified external downloader.
-                                     Currently supports
-                                     aria2c,avconv,axel,curl,ffmpeg,httpie,wget
-    --external-downloader-args ARGS  Give these arguments to the external
-                                     downloader
+    -r, --limit-rate RATE                Maximum download rate in bytes per
+                                         second (e.g. 50K or 4.2M)
+    -R, --retries RETRIES                Number of retries (default is 10), or
+                                         "infinite".
+    --fragment-retries RETRIES           Number of retries for a fragment
+                                         (default is 10), or "infinite" (DASH,
+                                         hlsnative and ISM)
+    --skip-unavailable-fragments         Skip unavailable fragments (DASH,
+                                         hlsnative and ISM)
+    --abort-on-unavailable-fragment      Abort downloading when some fragment is
+                                         not available
+    --keep-fragments                     Keep downloaded fragments on disk after
+                                         downloading is finished; fragments are
+                                         erased by default
+    --buffer-size SIZE                   Size of download buffer (e.g. 1024 or
+                                         16K) (default is 1024)
+    --no-resize-buffer                   Do not automatically adjust the buffer
+                                         size. By default, the buffer size is
+                                         automatically resized from an initial
+                                         value of SIZE.
+    --http-chunk-size SIZE               Size of a chunk for chunk-based HTTP
+                                         downloading (e.g. 10485760 or 10M)
+                                         (default is disabled). May be useful
+                                         for bypassing bandwidth throttling
+                                         imposed by a webserver (experimental)
+    --playlist-reverse                   Download playlist videos in reverse
+                                         order
+    --playlist-random                    Download playlist videos in random
+                                         order
+    --xattr-set-filesize                 Set file xattribute ytdl.filesize with
+                                         expected file size
+    --hls-prefer-native                  Use the native HLS downloader instead
+                                         of ffmpeg
+    --hls-prefer-ffmpeg                  Use ffmpeg instead of the native HLS
+                                         downloader
+    --hls-use-mpegts                     Use the mpegts container for HLS
+                                         videos, allowing to play the video
+                                         while downloading (some players may not
+                                         be able to play it)
+    --external-downloader COMMAND        Use the specified external downloader.
+                                         Currently supports aria2c,avconv,axel,c
+                                         url,ffmpeg,httpie,wget
+    --external-downloader-args ARGS      Give these arguments to the external
+                                         downloader
 
 ## Filesystem Options:
-    -a, --batch-file FILE            File containing URLs to download ('-' for
-                                     stdin), one URL per line. Lines starting
-                                     with '#', ';' or ']' are considered as
-                                     comments and ignored.
-    --id                             Use only video ID in file name
-    -o, --output TEMPLATE            Output filename template, see the "OUTPUT
-                                     TEMPLATE" for all the info
-    --autonumber-start NUMBER        Specify the start value for %(autonumber)s
-                                     (default is 1)
-    --restrict-filenames             Restrict filenames to only ASCII
-                                     characters, and avoid "&" and spaces in
-                                     filenames
-    -w, --no-overwrites              Do not overwrite files
-    -c, --continue                   Force resume of partially downloaded files.
-                                     By default, youtube-dl will resume
-                                     downloads if possible.
-    --no-continue                    Do not resume partially downloaded files
-                                     (restart from beginning)
-    --no-part                        Do not use .part files - write directly
-                                     into output file
-    --no-mtime                       Do not use the Last-modified header to set
-                                     the file modification time
-    --write-description              Write video description to a .description
-                                     file
-    --write-info-json                Write video metadata to a .info.json file
-    --write-annotations              Write video annotations to a
-                                     .annotations.xml file
-    --load-info-json FILE            JSON file containing the video information
-                                     (created with the "--write-info-json"
-                                     option)
-    --cookies FILE                   File to read cookies from and dump cookie
-                                     jar in
-    --cache-dir DIR                  Location in the filesystem where youtube-dl
-                                     can store some downloaded information
-                                     permanently. By default
-                                     $XDG_CACHE_HOME/youtube-dl or
-                                     ~/.cache/youtube-dl . At the moment, only
-                                     YouTube player files (for videos with
-                                     obfuscated signatures) are cached, but that
-                                     may change.
-    --no-cache-dir                   Disable filesystem caching
-    --rm-cache-dir                   Delete all filesystem cache files
+    -a, --batch-file FILE                File containing URLs to download ('-'
+                                         for stdin), one URL per line. Lines
+                                         starting with '#', ';' or ']' are
+                                         considered as comments and ignored.
+    --id                                 Use only video ID in file name
+    -o, --output TEMPLATE                Output filename template, see the
+                                         "OUTPUT TEMPLATE" for all the info
+    --output-na-placeholder PLACEHOLDER  Placeholder value for unavailable meta
+                                         fields in output filename template
+                                         (default is "NA")
+    --autonumber-start NUMBER            Specify the start value for
+                                         %(autonumber)s (default is 1)
+    --restrict-filenames                 Restrict filenames to only ASCII
+                                         characters, and avoid "&" and spaces in
+                                         filenames
+    -w, --no-overwrites                  Do not overwrite files
+    -c, --continue                       Force resume of partially downloaded
+                                         files. By default, youtube-dl will
+                                         resume downloads if possible.
+    --no-continue                        Do not resume partially downloaded
+                                         files (restart from beginning)
+    --no-part                            Do not use .part files - write directly
+                                         into output file
+    --no-mtime                           Do not use the Last-modified header to
+                                         set the file modification time
+    --write-description                  Write video description to a
+                                         .description file
+    --write-info-json                    Write video metadata to a .info.json
+                                         file
+    --write-annotations                  Write video annotations to a
+                                         .annotations.xml file
+    --load-info-json FILE                JSON file containing the video
+                                         information (created with the "--write-
+                                         info-json" option)
+    --cookies FILE                       File to read cookies from and dump
+                                         cookie jar in
+    --cache-dir DIR                      Location in the filesystem where
+                                         youtube-dl can store some downloaded
+                                         information permanently. By default
+                                         $XDG_CACHE_HOME/youtube-dl or
+                                         ~/.cache/youtube-dl . At the moment,
+                                         only YouTube player files (for videos
+                                         with obfuscated signatures) are cached,
+                                         but that may change.
+    --no-cache-dir                       Disable filesystem caching
+    --rm-cache-dir                       Delete all filesystem cache files
 
 ## Thumbnail images:
-    --write-thumbnail                Write thumbnail image to disk
-    --write-all-thumbnails           Write all thumbnail image formats to disk
-    --list-thumbnails                Simulate and list all available thumbnail
-                                     formats
+    --write-thumbnail                    Write thumbnail image to disk
+    --write-all-thumbnails               Write all thumbnail image formats to
+                                         disk
+    --list-thumbnails                    Simulate and list all available
+                                         thumbnail formats
 
 ## Verbosity / Simulation Options:
-    -q, --quiet                      Activate quiet mode
-    --no-warnings                    Ignore warnings
-    -s, --simulate                   Do not download the video and do not write
-                                     anything to disk
-    --skip-download                  Do not download the video
-    -g, --get-url                    Simulate, quiet but print URL
-    -e, --get-title                  Simulate, quiet but print title
-    --get-id                         Simulate, quiet but print id
-    --get-thumbnail                  Simulate, quiet but print thumbnail URL
-    --get-description                Simulate, quiet but print video description
-    --get-duration                   Simulate, quiet but print video length
-    --get-filename                   Simulate, quiet but print output filename
-    --get-format                     Simulate, quiet but print output format
-    -j, --dump-json                  Simulate, quiet but print JSON information.
-                                     See the "OUTPUT TEMPLATE" for a description
-                                     of available keys.
-    -J, --dump-single-json           Simulate, quiet but print JSON information
-                                     for each command-line argument. If the URL
-                                     refers to a playlist, dump the whole
-                                     playlist information in a single line.
-    --print-json                     Be quiet and print the video information as
-                                     JSON (video is still being downloaded).
-    --newline                        Output progress bar as new lines
-    --no-progress                    Do not print progress bar
-    --console-title                  Display progress in console titlebar
-    -v, --verbose                    Print various debugging information
-    --dump-pages                     Print downloaded pages encoded using base64
-                                     to debug problems (very verbose)
-    --write-pages                    Write downloaded intermediary pages to
-                                     files in the current directory to debug
-                                     problems
-    --print-traffic                  Display sent and read HTTP traffic
-    -C, --call-home                  Contact the youtube-dl server for debugging
-    --no-call-home                   Do NOT contact the youtube-dl server for
-                                     debugging
+    -q, --quiet                          Activate quiet mode
+    --no-warnings                        Ignore warnings
+    -s, --simulate                       Do not download the video and do not
+                                         write anything to disk
+    --skip-download                      Do not download the video
+    -g, --get-url                        Simulate, quiet but print URL
+    -e, --get-title                      Simulate, quiet but print title
+    --get-id                             Simulate, quiet but print id
+    --get-thumbnail                      Simulate, quiet but print thumbnail URL
+    --get-description                    Simulate, quiet but print video
+                                         description
+    --get-duration                       Simulate, quiet but print video length
+    --get-filename                       Simulate, quiet but print output
+                                         filename
+    --get-format                         Simulate, quiet but print output format
+    -j, --dump-json                      Simulate, quiet but print JSON
+                                         information. See the "OUTPUT TEMPLATE"
+                                         for a description of available keys.
+    -J, --dump-single-json               Simulate, quiet but print JSON
+                                         information for each command-line
+                                         argument. If the URL refers to a
+                                         playlist, dump the whole playlist
+                                         information in a single line.
+    --print-json                         Be quiet and print the video
+                                         information as JSON (video is still
+                                         being downloaded).
+    --newline                            Output progress bar as new lines
+    --no-progress                        Do not print progress bar
+    --console-title                      Display progress in console titlebar
+    -v, --verbose                        Print various debugging information
+    --dump-pages                         Print downloaded pages encoded using
+                                         base64 to debug problems (very verbose)
+    --write-pages                        Write downloaded intermediary pages to
+                                         files in the current directory to debug
+                                         problems
+    --print-traffic                      Display sent and read HTTP traffic
+    -C, --call-home                      Contact the youtube-dl server for
+                                         debugging
+    --no-call-home                       Do NOT contact the youtube-dl server
+                                         for debugging
 
 ## Workarounds:
-    --encoding ENCODING              Force the specified encoding (experimental)
-    --no-check-certificate           Suppress HTTPS certificate validation
-    --prefer-insecure                Use an unencrypted connection to retrieve
-                                     information about the video. (Currently
-                                     supported only for YouTube)
-    --user-agent UA                  Specify a custom user agent
-    --referer URL                    Specify a custom referer, use if the video
-                                     access is restricted to one domain
-    --add-header FIELD:VALUE         Specify a custom HTTP header and its value,
-                                     separated by a colon ':'. You can use this
-                                     option multiple times
-    --bidi-workaround                Work around terminals that lack
-                                     bidirectional text support. Requires bidiv
-                                     or fribidi executable in PATH
-    --sleep-interval SECONDS         Number of seconds to sleep before each
-                                     download when used alone or a lower bound
-                                     of a range for randomized sleep before each
-                                     download (minimum possible number of
-                                     seconds to sleep) when used along with
-                                     --max-sleep-interval.
-    --max-sleep-interval SECONDS     Upper bound of a range for randomized sleep
-                                     before each download (maximum possible
-                                     number of seconds to sleep). Must only be
-                                     used along with --min-sleep-interval.
+    --encoding ENCODING                  Force the specified encoding
+                                         (experimental)
+    --no-check-certificate               Suppress HTTPS certificate validation
+    --prefer-insecure                    Use an unencrypted connection to
+                                         retrieve information about the video.
+                                         (Currently supported only for YouTube)
+    --user-agent UA                      Specify a custom user agent
+    --referer URL                        Specify a custom referer, use if the
+                                         video access is restricted to one
+                                         domain
+    --add-header FIELD:VALUE             Specify a custom HTTP header and its
+                                         value, separated by a colon ':'. You
+                                         can use this option multiple times
+    --bidi-workaround                    Work around terminals that lack
+                                         bidirectional text support. Requires
+                                         bidiv or fribidi executable in PATH
+    --sleep-interval SECONDS             Number of seconds to sleep before each
+                                         download when used alone or a lower
+                                         bound of a range for randomized sleep
+                                         before each download (minimum possible
+                                         number of seconds to sleep) when used
+                                         along with --max-sleep-interval.
+    --max-sleep-interval SECONDS         Upper bound of a range for randomized
+                                         sleep before each download (maximum
+                                         possible number of seconds to sleep).
+                                         Must only be used along with --min-
+                                         sleep-interval.
 
 ## Video Format Options:
-    -f, --format FORMAT              Video format code, see the "FORMAT
-                                     SELECTION" for all the info
-    --all-formats                    Download all available video formats
-    --prefer-free-formats            Prefer free video formats unless a specific
-                                     one is requested
-    -F, --list-formats               List all available formats of requested
-                                     videos
-    --youtube-skip-dash-manifest     Do not download the DASH manifests and
-                                     related data on YouTube videos
-    --merge-output-format FORMAT     If a merge is required (e.g.
-                                     bestvideo+bestaudio), output to given
-                                     container format. One of mkv, mp4, ogg,
-                                     webm, flv. Ignored if no merge is required
+    -f, --format FORMAT                  Video format code, see the "FORMAT
+                                         SELECTION" for all the info
+    --all-formats                        Download all available video formats
+    --prefer-free-formats                Prefer free video formats unless a
+                                         specific one is requested
+    -F, --list-formats                   List all available formats of requested
+                                         videos
+    --youtube-skip-dash-manifest         Do not download the DASH manifests and
+                                         related data on YouTube videos
+    --merge-output-format FORMAT         If a merge is required (e.g.
+                                         bestvideo+bestaudio), output to given
+                                         container format. One of mkv, mp4, ogg,
+                                         webm, flv. Ignored if no merge is
+                                         required
 
 ## Subtitle Options:
-    --write-sub                      Write subtitle file
-    --write-auto-sub                 Write automatically generated subtitle file
-                                     (YouTube only)
-    --all-subs                       Download all the available subtitles of the
-                                     video
-    --list-subs                      List all available subtitles for the video
-    --sub-format FORMAT              Subtitle format, accepts formats
-                                     preference, for example: "srt" or
-                                     "ass/srt/best"
-    --sub-lang LANGS                 Languages of the subtitles to download
-                                     (optional) separated by commas, use --list-
-                                     subs for available language tags
+    --write-sub                          Write subtitle file
+    --write-auto-sub                     Write automatically generated subtitle
+                                         file (YouTube only)
+    --all-subs                           Download all the available subtitles of
+                                         the video
+    --list-subs                          List all available subtitles for the
+                                         video
+    --sub-format FORMAT                  Subtitle format, accepts formats
+                                         preference, for example: "srt" or
+                                         "ass/srt/best"
+    --sub-lang LANGS                     Languages of the subtitles to download
+                                         (optional) separated by commas, use
+                                         --list-subs for available language tags
 
 ## Authentication Options:
-    -u, --username USERNAME          Login with this account ID
-    -p, --password PASSWORD          Account password. If this option is left
-                                     out, youtube-dl will ask interactively.
-    -2, --twofactor TWOFACTOR        Two-factor authentication code
-    -n, --netrc                      Use .netrc authentication data
-    --video-password PASSWORD        Video password (vimeo, youku)
+    -u, --username USERNAME              Login with this account ID
+    -p, --password PASSWORD              Account password. If this option is
+                                         left out, youtube-dl will ask
+                                         interactively.
+    -2, --twofactor TWOFACTOR            Two-factor authentication code
+    -n, --netrc                          Use .netrc authentication data
+    --video-password PASSWORD            Video password (vimeo, youku)
 
 ## Adobe Pass Options:
-    --ap-mso MSO                     Adobe Pass multiple-system operator (TV
-                                     provider) identifier, use --ap-list-mso for
-                                     a list of available MSOs
-    --ap-username USERNAME           Multiple-system operator account login
-    --ap-password PASSWORD           Multiple-system operator account password.
-                                     If this option is left out, youtube-dl will
-                                     ask interactively.
-    --ap-list-mso                    List all supported multiple-system
-                                     operators
+    --ap-mso MSO                         Adobe Pass multiple-system operator (TV
+                                         provider) identifier, use --ap-list-mso
+                                         for a list of available MSOs
+    --ap-username USERNAME               Multiple-system operator account login
+    --ap-password PASSWORD               Multiple-system operator account
+                                         password. If this option is left out,
+                                         youtube-dl will ask interactively.
+    --ap-list-mso                        List all supported multiple-system
+                                         operators
 
 ## Post-processing Options:
-    -x, --extract-audio              Convert video files to audio-only files
-                                     (requires ffmpeg or avconv and ffprobe or
-                                     avprobe)
-    --audio-format FORMAT            Specify audio format: "best", "aac",
-                                     "flac", "mp3", "m4a", "opus", "vorbis", or
-                                     "wav"; "best" by default; No effect without
-                                     -x
-    --audio-quality QUALITY          Specify ffmpeg/avconv audio quality, insert
-                                     a value between 0 (better) and 9 (worse)
-                                     for VBR or a specific bitrate like 128K
-                                     (default 5)
-    --recode-video FORMAT            Encode the video to another format if
-                                     necessary (currently supported:
-                                     mp4|flv|ogg|webm|mkv|avi)
-    --postprocessor-args ARGS        Give these arguments to the postprocessor
-    -k, --keep-video                 Keep the video file on disk after the post-
-                                     processing; the video is erased by default
-    --no-post-overwrites             Do not overwrite post-processed files; the
-                                     post-processed files are overwritten by
-                                     default
-    --embed-subs                     Embed subtitles in the video (only for mp4,
-                                     webm and mkv videos)
-    --embed-thumbnail                Embed thumbnail in the audio as cover art
-    --add-metadata                   Write metadata to the video file
-    --metadata-from-title FORMAT     Parse additional metadata like song title /
-                                     artist from the video title. The format
-                                     syntax is the same as --output. Regular
-                                     expression with named capture groups may
-                                     also be used. The parsed parameters replace
-                                     existing values. Example: --metadata-from-
-                                     title "%(artist)s - %(title)s" matches a
-                                     title like "Coldplay - Paradise". Example
-                                     (regex): --metadata-from-title
-                                     "(?P<artist>.+?) - (?P<title>.+)"
-    --xattrs                         Write metadata to the video file's xattrs
-                                     (using dublin core and xdg standards)
-    --fixup POLICY                   Automatically correct known faults of the
-                                     file. One of never (do nothing), warn (only
-                                     emit a warning), detect_or_warn (the
-                                     default; fix file if we can, warn
-                                     otherwise)
-    --prefer-avconv                  Prefer avconv over ffmpeg for running the
-                                     postprocessors
-    --prefer-ffmpeg                  Prefer ffmpeg over avconv for running the
-                                     postprocessors (default)
-    --ffmpeg-location PATH           Location of the ffmpeg/avconv binary;
-                                     either the path to the binary or its
-                                     containing directory.
-    --exec CMD                       Execute a command on the file after
-                                     downloading and post-processing, similar to
-                                     find's -exec syntax. Example: --exec 'adb
-                                     push {} /sdcard/Music/ && rm {}'
-    --convert-subs FORMAT            Convert the subtitles to other format
-                                     (currently supported: srt|ass|vtt|lrc)
+    -x, --extract-audio                  Convert video files to audio-only files
+                                         (requires ffmpeg/avconv and
+                                         ffprobe/avprobe)
+    --audio-format FORMAT                Specify audio format: "best", "aac",
+                                         "flac", "mp3", "m4a", "opus", "vorbis",
+                                         or "wav"; "best" by default; No effect
+                                         without -x
+    --audio-quality QUALITY              Specify ffmpeg/avconv audio quality,
+                                         insert a value between 0 (better) and 9
+                                         (worse) for VBR or a specific bitrate
+                                         like 128K (default 5)
+    --recode-video FORMAT                Encode the video to another format if
+                                         necessary (currently supported:
+                                         mp4|flv|ogg|webm|mkv|avi)
+    --postprocessor-args ARGS            Give these arguments to the
+                                         postprocessor
+    -k, --keep-video                     Keep the video file on disk after the
+                                         post-processing; the video is erased by
+                                         default
+    --no-post-overwrites                 Do not overwrite post-processed files;
+                                         the post-processed files are
+                                         overwritten by default
+    --embed-subs                         Embed subtitles in the video (only for
+                                         mp4, webm and mkv videos)
+    --embed-thumbnail                    Embed thumbnail in the audio as cover
+                                         art
+    --add-metadata                       Write metadata to the video file
+    --metadata-from-title FORMAT         Parse additional metadata like song
+                                         title / artist from the video title.
+                                         The format syntax is the same as
+                                         --output. Regular expression with named
+                                         capture groups may also be used. The
+                                         parsed parameters replace existing
+                                         values. Example: --metadata-from-title
+                                         "%(artist)s - %(title)s" matches a
+                                         title like "Coldplay - Paradise".
+                                         Example (regex): --metadata-from-title
+                                         "(?P<artist>.+?) - (?P<title>.+)"
+    --xattrs                             Write metadata to the video file's
+                                         xattrs (using dublin core and xdg
+                                         standards)
+    --fixup POLICY                       Automatically correct known faults of
+                                         the file. One of never (do nothing),
+                                         warn (only emit a warning),
+                                         detect_or_warn (the default; fix file
+                                         if we can, warn otherwise)
+    --prefer-avconv                      Prefer avconv over ffmpeg for running
+                                         the postprocessors
+    --prefer-ffmpeg                      Prefer ffmpeg over avconv for running
+                                         the postprocessors (default)
+    --ffmpeg-location PATH               Location of the ffmpeg/avconv binary;
+                                         either the path to the binary or its
+                                         containing directory.
+    --exec CMD                           Execute a command on the file after
+                                         downloading and post-processing,
+                                         similar to find's -exec syntax.
+                                         Example: --exec 'adb push {}
+                                         /sdcard/Music/ && rm {}'
+    --convert-subs FORMAT                Convert the subtitles to other format
+                                         (currently supported: srt|ass|vtt|lrc)
 
 # CONFIGURATION
 
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index aa8026a32..13bac6e27 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -46,10 +46,11 @@
  - **Amara**
  - **AMCNetworks**
  - **AmericasTestKitchen**
+ - **AmericasTestKitchenSeason**
  - **anderetijden**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl
  - **AnimeOnDemand**
  - **Anvato**
- - **aol.com**
+ - **aol.com**: Yahoo screen and movies
  - **APA**
  - **Aparat**
  - **AppleConnect**
@@ -192,8 +193,6 @@
  - **CNNArticle**
  - **CNNBlogs**
  - **ComedyCentral**
- - **ComedyCentralFullEpisodes**
- - **ComedyCentralShortname**
  - **ComedyCentralTV**
  - **CondeNast**: Condé Nast media group: Allure, Architectural Digest, Ars Technica, Bon Appétit, Brides, Condé Nast, Condé Nast Traveler, Details, Epicurious, GQ, Glamour, Golf Digest, SELF, Teen Vogue, The New Yorker, Vanity Fair, Vogue, W Magazine, WIRED
  - **CONtv**
@@ -506,6 +505,9 @@
  - **Mgoon**
  - **MGTV**: 芒果TV
  - **MiaoPai**
+ - **minds**
+ - **minds:channel**
+ - **minds:group**
  - **MinistryGrid**
  - **Minoto**
  - **miomio.tv**
@@ -859,6 +861,8 @@
  - **Sport5**
  - **SportBox**
  - **SportDeutschland**
+ - **spotify**
+ - **spotify:show**
  - **Spreaker**
  - **SpreakerPage**
  - **SpreakerShow**
@@ -940,12 +944,13 @@
  - **TNAFlixNetworkEmbed**
  - **toggle**
  - **ToonGoggles**
- - **Tosh**: Tosh.0
  - **tou.tv**
  - **Toypics**: Toypics video
  - **ToypicsUser**: Toypics user profile
  - **TrailerAddict** (Currently broken)
  - **Trilulilu**
+ - **Trovo**
+ - **TrovoVod**
  - **TruNews**
  - **TruTV**
  - **Tube8**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index ac7242abb..080460d50 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.01.16'
+__version__ = '2021.01.24'

From b63981e85095542e056d7180496de4a2d85ddf9e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Jan 2021 18:11:20 +0700
Subject: [PATCH 151/860] release 2021.01.24.1

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 youtube_dl/version.py                            | 2 +-
 6 files changed, 13 insertions(+), 13 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 935806b5e..2dde97a2c 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.01.24**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.24.1**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.01.24
+ [debug] youtube-dl version 2021.01.24.1
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index 453a5e147..c520d1ee0 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.01.24**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.24.1**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 07094c10d..4aacd3bdc 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.01.24**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.24.1**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index b234f8ccd..91bbed506 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.01.24**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.24.1**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.01.24
+ [debug] youtube-dl version 2021.01.24.1
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index 557e59ca5..a0a2c989a 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.01.24**
+- [ ] I've verified that I'm running youtube-dl version **2021.01.24.1**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 080460d50..c52f1d9ca 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.01.24'
+__version__ = '2021.01.24.1'

From 811a183eb6a7dabcc8600f6bf9dc4aac11e72b26 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 24 Jan 2021 18:15:32 +0700
Subject: [PATCH 152/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/ChangeLog b/ChangeLog
index 9b2f38b25..7f2e0aad1 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version 2021.01.24
+version 2021.01.24.1
 
 Core
 * Introduce --output-na-placeholder (#27896)

From ea399a53eb9744c0b1530d72bf9d5e1e21ed3489 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adrian=20Heine=20n=C3=A9=20Lang?= <mail@adrianheine.de>
Date: Mon, 25 Jan 2021 09:25:57 +0100
Subject: [PATCH 153/860] [ADN] Implement login (#27937)

closes #17091
closes #27841
---
 youtube_dl/extractor/adn.py | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/adn.py b/youtube_dl/extractor/adn.py
index d611ee237..40111586d 100644
--- a/youtube_dl/extractor/adn.py
+++ b/youtube_dl/extractor/adn.py
@@ -26,6 +26,7 @@ from ..utils import (
     strip_or_none,
     try_get,
     unified_strdate,
+    urlencode_postdata,
 )
 
 
@@ -51,9 +52,11 @@ class ADNIE(InfoExtractor):
         }
     }
 
+    _NETRC_MACHINE = 'animedigitalnetwork'
     _BASE_URL = 'http://animedigitalnetwork.fr'
     _API_BASE_URL = 'https://gw.api.animedigitalnetwork.fr/'
     _PLAYER_BASE_URL = _API_BASE_URL + 'player/'
+    _HEADERS = {}
     _RSA_KEY = (0x9B42B08905199A5CCE2026274399CA560ECB209EE9878A708B1C0812E1BB8CB5D1FB7441861147C1A1F2F3A0476DD63A9CAC20D3E983613346850AA6CB38F16DC7D720FD7D86FC6E5B3D5BBC72E14CD0BF9E869F2CEA2CCAD648F1DCE38F1FF916CEFB2D339B64AA0264372344BC775E265E8A852F88144AB0BD9AA06C1A4ABB, 65537)
     _POS_ALIGN_MAP = {
         'start': 1,
@@ -129,19 +132,32 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
             }])
         return subtitles
 
+    def _real_initialize(self):
+        username, password = self._get_login_info()
+        if username:
+            access_token = (self._download_json(
+                self._API_BASE_URL + 'authentication/login', None,
+                'Logging in', errnote='Unable to log in', fatal=False,
+                data=urlencode_postdata({
+                    'password': password,
+                    'rememberMe': False,
+                    'source': 'Web',
+                    'username': username,
+                })) or {}).get('accessToken')
+            if access_token:
+                self._HEADERS = {'authorization': 'Bearer ' + access_token}
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         video_base_url = self._PLAYER_BASE_URL + 'video/%s/' % video_id
         player = self._download_json(
             video_base_url + 'configuration', video_id,
-            'Downloading player config JSON metadata')['player']
+            'Downloading player config JSON metadata', headers=self._HEADERS)['player']
         options = player['options']
 
         user = options['user']
         if not user.get('hasAccess'):
-            raise ExtractorError(
-                'This video is only available for paying users', expected=True)
-            # self.raise_login_required() # FIXME: Login is not implemented
+            self.raise_login_required()
 
         token = self._download_json(
             user.get('refreshTokenUrl') or (self._PLAYER_BASE_URL + 'refresh/token'),

From 57f2488bbe4345eee545488e937e0f8426664f6f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 25 Jan 2021 09:20:48 +0100
Subject: [PATCH 154/860] [zype] fix uplynk id extraction(closes #27956)

---
 youtube_dl/extractor/zype.py | 15 ++++++++++-----
 1 file changed, 10 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/zype.py b/youtube_dl/extractor/zype.py
index 5288f40d8..f20f953cb 100644
--- a/youtube_dl/extractor/zype.py
+++ b/youtube_dl/extractor/zype.py
@@ -87,11 +87,16 @@ class ZypeIE(InfoExtractor):
                 r'(["\'])(?P<url>(?:(?!\1).)+\.m3u8(?:(?!\1).)*)\1',
                 body, 'm3u8 url', group='url', default=None)
             if not m3u8_url:
-                source = self._parse_json(self._search_regex(
-                    r'(?s)sources\s*:\s*\[\s*({.+?})\s*\]', body,
-                    'source'), video_id, js_to_json)
-                if source.get('integration') == 'verizon-media':
-                    m3u8_url = 'https://content.uplynk.com/%s.m3u8' % source['id']
+                source = self._search_regex(
+                    r'(?s)sources\s*:\s*\[\s*({.+?})\s*\]', body, 'source')
+
+                def get_attr(key):
+                    return self._search_regex(
+                        r'\b%s\s*:\s*([\'"])(?P<val>(?:(?!\1).)+)\1' % key,
+                        source, key, group='val')
+
+                if get_attr('integration') == 'verizon-media':
+                    m3u8_url = 'https://content.uplynk.com/%s.m3u8' % get_attr('id')
             formats = self._extract_m3u8_formats(
                 m3u8_url, video_id, 'mp4', 'm3u8_native', m3u8_id='hls')
             text_tracks = self._search_regex(

From 55bb3556c8c5ec088ef88edbdef925860e0926dc Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 25 Jan 2021 09:28:24 +0100
Subject: [PATCH 155/860] [adn] improve login warning reporting

---
 youtube_dl/extractor/adn.py | 20 +++++++++++++++-----
 1 file changed, 15 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/adn.py b/youtube_dl/extractor/adn.py
index 40111586d..a55ebbcbd 100644
--- a/youtube_dl/extractor/adn.py
+++ b/youtube_dl/extractor/adn.py
@@ -57,6 +57,7 @@ class ADNIE(InfoExtractor):
     _API_BASE_URL = 'https://gw.api.animedigitalnetwork.fr/'
     _PLAYER_BASE_URL = _API_BASE_URL + 'player/'
     _HEADERS = {}
+    _LOGIN_ERR_MESSAGE = 'Unable to log in'
     _RSA_KEY = (0x9B42B08905199A5CCE2026274399CA560ECB209EE9878A708B1C0812E1BB8CB5D1FB7441861147C1A1F2F3A0476DD63A9CAC20D3E983613346850AA6CB38F16DC7D720FD7D86FC6E5B3D5BBC72E14CD0BF9E869F2CEA2CCAD648F1DCE38F1FF916CEFB2D339B64AA0264372344BC775E265E8A852F88144AB0BD9AA06C1A4ABB, 65537)
     _POS_ALIGN_MAP = {
         'start': 1,
@@ -134,10 +135,12 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
 
     def _real_initialize(self):
         username, password = self._get_login_info()
-        if username:
+        if not username:
+            return
+        try:
             access_token = (self._download_json(
                 self._API_BASE_URL + 'authentication/login', None,
-                'Logging in', errnote='Unable to log in', fatal=False,
+                'Logging in', self._LOGIN_ERR_MESSAGE, fatal=False,
                 data=urlencode_postdata({
                     'password': password,
                     'rememberMe': False,
@@ -146,13 +149,21 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
                 })) or {}).get('accessToken')
             if access_token:
                 self._HEADERS = {'authorization': 'Bearer ' + access_token}
+        except ExtractorError as e:
+            message = None
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
+                resp = self._parse_json(
+                    e.cause.read().decode(), None, fatal=False) or {}
+                message = resp.get('message') or resp.get('code')
+            self.report_warning(message or self._LOGIN_ERR_MESSAGE)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         video_base_url = self._PLAYER_BASE_URL + 'video/%s/' % video_id
         player = self._download_json(
             video_base_url + 'configuration', video_id,
-            'Downloading player config JSON metadata', headers=self._HEADERS)['player']
+            'Downloading player config JSON metadata',
+            headers=self._HEADERS)['player']
         options = player['options']
 
         user = options['user']
@@ -204,8 +215,7 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
                 message = error.get('message')
                 if e.cause.code == 403 and error.get('code') == 'player-bad-geolocation-country':
                     self.raise_geo_restricted(msg=message)
-                else:
-                    raise ExtractorError(message)
+                raise ExtractorError(message)
         else:
             raise ExtractorError('Giving up retrying')
 

From 395981288ba0b2e1afabd4e595cb9c959ef62356 Mon Sep 17 00:00:00 2001
From: tpikonen <tpikonen@gmail.com>
Date: Mon, 25 Jan 2021 15:43:41 +0200
Subject: [PATCH 156/860] [tv2] Add support for mtvuutiset.fi (#27744)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/tv2.py        | 29 +++++++++++++++++++++++++++++
 2 files changed, 30 insertions(+)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index ef57f5556..2331b0e15 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1260,6 +1260,7 @@ from .tv2 import (
     TV2IE,
     TV2ArticleIE,
     KatsomoIE,
+    MTVuutisetIE,
 )
 from .tv2dk import (
     TV2DKIE,
diff --git a/youtube_dl/extractor/tv2.py b/youtube_dl/extractor/tv2.py
index 4a19b9be6..42a9af126 100644
--- a/youtube_dl/extractor/tv2.py
+++ b/youtube_dl/extractor/tv2.py
@@ -190,3 +190,32 @@ class KatsomoIE(TV2IE):
     _API_DOMAIN = 'api.katsomo.fi'
     _PROTOCOLS = ('HLS', 'MPD')
     _GEO_COUNTRIES = ['FI']
+
+
+class MTVuutisetIE(KatsomoIE):
+    _VALID_URL = r'https?://(?:www\.)mtvuutiset\.fi/(?:artikkeli/[0-9a-z-]+/|video/prog)(?P<id>\d+)'
+    _TEST = {
+        'url': 'https://www.mtvuutiset.fi/artikkeli/tallaisia-vaurioita-viking-amorellassa-on-useamman-osaston-alla-vetta/7931384',
+        'info_dict': {
+            'id': '1311159',
+            'ext': 'mp4',
+            'title': 'MTV Uutiset Live',
+            'description': 'Viking Amorellan matkustajien evakuointi on alkanut – tältä operaatio näyttää laivalla',
+            'timestamp': 1600608966,
+            'upload_date': '20200920',
+            'duration': 153.7886666,
+            'view_count': int,
+            'categories': list,
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        art_id = self._match_id(url)
+        webpage = self._download_webpage(url, art_id)
+        video_id = self._html_search_regex(
+            r'<div class=\'player-container\' .*data-katsomoid="(.+?)"', webpage, 'video_id')
+        return self.url_result("http://mtv.fi/a/0/a/%s" % video_id, video_id=video_id, ie="Katsomo")

From 286e5d6724f86ace38f6ccf0e33c511eeb1ab65d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 25 Jan 2021 14:46:04 +0100
Subject: [PATCH 157/860] [tv2] improve MTV Uutiset Article extraction

---
 youtube_dl/extractor/extractors.py |  2 +-
 youtube_dl/extractor/tv2.py        | 73 ++++++++++++++++++++----------
 2 files changed, 51 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 2331b0e15..c554a8504 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1260,7 +1260,7 @@ from .tv2 import (
     TV2IE,
     TV2ArticleIE,
     KatsomoIE,
-    MTVuutisetIE,
+    MTVUutisetArticleIE,
 )
 from .tv2dk import (
     TV2DKIE,
diff --git a/youtube_dl/extractor/tv2.py b/youtube_dl/extractor/tv2.py
index 42a9af126..334b7d540 100644
--- a/youtube_dl/extractor/tv2.py
+++ b/youtube_dl/extractor/tv2.py
@@ -20,7 +20,7 @@ from ..utils import (
 
 class TV2IE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?tv2\.no/v/(?P<id>\d+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'http://www.tv2.no/v/916509/',
         'info_dict': {
             'id': '916509',
@@ -33,7 +33,7 @@ class TV2IE(InfoExtractor):
             'view_count': int,
             'categories': list,
         },
-    }
+    }]
     _API_DOMAIN = 'sumo.tv2.no'
     _PROTOCOLS = ('HDS', 'HLS', 'DASH')
     _GEO_COUNTRIES = ['NO']
@@ -42,6 +42,12 @@ class TV2IE(InfoExtractor):
         video_id = self._match_id(url)
         api_base = 'http://%s/api/web/asset/%s' % (self._API_DOMAIN, video_id)
 
+        asset = self._download_json(
+            api_base + '.json', video_id,
+            'Downloading metadata JSON')['asset']
+        title = asset.get('subtitle') or asset['title']
+        is_live = asset.get('live') is True
+
         formats = []
         format_urls = []
         for protocol in self._PROTOCOLS:
@@ -81,7 +87,8 @@ class TV2IE(InfoExtractor):
                 elif ext == 'm3u8':
                     if not data.get('drmProtected'):
                         formats.extend(self._extract_m3u8_formats(
-                            video_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                            video_url, video_id, 'mp4',
+                            'm3u8' if is_live else 'm3u8_native',
                             m3u8_id=format_id, fatal=False))
                 elif ext == 'mpd':
                     formats.extend(self._extract_mpd_formats(
@@ -99,11 +106,6 @@ class TV2IE(InfoExtractor):
             raise ExtractorError('This video is DRM protected.', expected=True)
         self._sort_formats(formats)
 
-        asset = self._download_json(
-            api_base + '.json', video_id,
-            'Downloading metadata JSON')['asset']
-        title = asset['title']
-
         thumbnails = [{
             'id': thumbnail.get('@type'),
             'url': thumbnail.get('url'),
@@ -112,7 +114,7 @@ class TV2IE(InfoExtractor):
         return {
             'id': video_id,
             'url': video_url,
-            'title': title,
+            'title': self._live_title(title) if is_live else title,
             'description': strip_or_none(asset.get('description')),
             'thumbnails': thumbnails,
             'timestamp': parse_iso8601(asset.get('createTime')),
@@ -120,6 +122,7 @@ class TV2IE(InfoExtractor):
             'view_count': int_or_none(asset.get('views')),
             'categories': asset.get('keywords', '').split(','),
             'formats': formats,
+            'is_live': is_live,
         }
 
 
@@ -168,13 +171,13 @@ class TV2ArticleIE(InfoExtractor):
 
 
 class KatsomoIE(TV2IE):
-    _VALID_URL = r'https?://(?:www\.)?(?:katsomo|mtv)\.fi/(?:#!/)?(?:[^/]+/[0-9a-z-]+-\d+/[0-9a-z-]+-|[^/]+/\d+/[^/]+/)(?P<id>\d+)'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?(?:katsomo|mtv(uutiset)?)\.fi/(?:sarja/[0-9a-z-]+-\d+/[0-9a-z-]+-|(?:#!/)?jakso/(?:\d+/[^/]+/)?|video/prog)(?P<id>\d+)'
+    _TESTS = [{
         'url': 'https://www.mtv.fi/sarja/mtv-uutiset-live-33001002003/lahden-pelicans-teki-kovan-ratkaisun-ville-nieminen-pihalle-1181321',
         'info_dict': {
             'id': '1181321',
             'ext': 'mp4',
-            'title': 'MTV Uutiset Live',
+            'title': 'Lahden Pelicans teki kovan ratkaisun – Ville Nieminen pihalle',
             'description': 'Päätöksen teki Pelicansin hallitus.',
             'timestamp': 1575116484,
             'upload_date': '20191130',
@@ -186,20 +189,29 @@ class KatsomoIE(TV2IE):
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }, {
+        'url': 'http://www.katsomo.fi/#!/jakso/33001005/studio55-fi/658521/jukka-kuoppamaki-tekee-yha-lauluja-vaikka-lentokoneessa',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.mtvuutiset.fi/video/prog1311159',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.katsomo.fi/#!/jakso/1311159',
+        'only_matching': True,
+    }]
     _API_DOMAIN = 'api.katsomo.fi'
     _PROTOCOLS = ('HLS', 'MPD')
     _GEO_COUNTRIES = ['FI']
 
 
-class MTVuutisetIE(KatsomoIE):
-    _VALID_URL = r'https?://(?:www\.)mtvuutiset\.fi/(?:artikkeli/[0-9a-z-]+/|video/prog)(?P<id>\d+)'
-    _TEST = {
+class MTVUutisetArticleIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)mtvuutiset\.fi/artikkeli/[^/]+/(?P<id>\d+)'
+    _TESTS = [{
         'url': 'https://www.mtvuutiset.fi/artikkeli/tallaisia-vaurioita-viking-amorellassa-on-useamman-osaston-alla-vetta/7931384',
         'info_dict': {
             'id': '1311159',
             'ext': 'mp4',
-            'title': 'MTV Uutiset Live',
+            'title': 'Viking Amorellan matkustajien evakuointi on alkanut – tältä operaatio näyttää laivalla',
             'description': 'Viking Amorellan matkustajien evakuointi on alkanut – tältä operaatio näyttää laivalla',
             'timestamp': 1600608966,
             'upload_date': '20200920',
@@ -211,11 +223,26 @@ class MTVuutisetIE(KatsomoIE):
             # m3u8 download
             'skip_download': True,
         },
-    }
+    }, {
+        # multiple Youtube embeds
+        'url': 'https://www.mtvuutiset.fi/artikkeli/50-vuotta-subarun-vastaiskua/6070962',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        art_id = self._match_id(url)
-        webpage = self._download_webpage(url, art_id)
-        video_id = self._html_search_regex(
-            r'<div class=\'player-container\' .*data-katsomoid="(.+?)"', webpage, 'video_id')
-        return self.url_result("http://mtv.fi/a/0/a/%s" % video_id, video_id=video_id, ie="Katsomo")
+        article_id = self._match_id(url)
+        article = self._download_json(
+            'http://api.mtvuutiset.fi/mtvuutiset/api/json/' + article_id,
+            article_id)
+
+        def entries():
+            for video in (article.get('videos') or []):
+                video_type = video.get('videotype')
+                video_url = video.get('url')
+                if not (video_url and video_type in ('katsomo', 'youtube')):
+                    continue
+                yield self.url_result(
+                    video_url, video_type.capitalize(), video.get('video_id'))
+
+        return self.playlist_result(
+            entries(), article_id, article.get('title'), article.get('description'))

From 0f7d413d5b2637e2fb091745ab4f70811a6cc600 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 25 Jan 2021 15:15:45 +0100
Subject: [PATCH 158/860] [tv4] relax _VALID_URL(closes #27964)

---
 youtube_dl/extractor/tv4.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tv4.py b/youtube_dl/extractor/tv4.py
index c498b0191..b73bab9a8 100644
--- a/youtube_dl/extractor/tv4.py
+++ b/youtube_dl/extractor/tv4.py
@@ -17,7 +17,7 @@ class TV4IE(InfoExtractor):
             tv4\.se/(?:[^/]+)/klipp/(?:.*)-|
             tv4play\.se/
             (?:
-                (?:program|barn)/(?:[^/]+/|(?:[^\?]+)\?video_id=)|
+                (?:program|barn)/(?:(?:[^/]+/){1,2}|(?:[^\?]+)\?video_id=)|
                 iframe/video/|
                 film/|
                 sport/|
@@ -65,6 +65,10 @@ class TV4IE(InfoExtractor):
         {
             'url': 'http://www.tv4play.se/program/farang/3922081',
             'only_matching': True,
+        },
+        {
+            'url': 'https://www.tv4play.se/program/nyheterna/avsnitt/13315940',
+            'only_matching': True,
         }
     ]
 

From d18f4419a72a01abc2cb45ef23f2400cd3eb5f43 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adrian=20Heine=20n=C3=A9=20Lang?= <mail@adrianheine.de>
Date: Tue, 26 Jan 2021 22:43:11 +0100
Subject: [PATCH 159/860] [AMP] Fix upload_date and timestamp extraction
 (#27970)

---
 youtube_dl/extractor/abcnews.py        |  2 ++
 youtube_dl/extractor/amp.py            |  3 ++-
 youtube_dl/extractor/bleacherreport.py | 10 ++++++++--
 3 files changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/abcnews.py b/youtube_dl/extractor/abcnews.py
index 8b407bf9c..64ea6e6ed 100644
--- a/youtube_dl/extractor/abcnews.py
+++ b/youtube_dl/extractor/abcnews.py
@@ -36,6 +36,8 @@ class AbcNewsVideoIE(AMPIE):
             'description': 'George Stephanopoulos goes one-on-one with Iranian Foreign Minister Dr. Javad Zarif.',
             'duration': 180,
             'thumbnail': r're:^https?://.*\.jpg$',
+            'timestamp': 1380454200,
+            'upload_date': '20130929',
         },
         'params': {
             # m3u8 download
diff --git a/youtube_dl/extractor/amp.py b/youtube_dl/extractor/amp.py
index 7ff098cfa..24c684cad 100644
--- a/youtube_dl/extractor/amp.py
+++ b/youtube_dl/extractor/amp.py
@@ -8,6 +8,7 @@ from ..utils import (
     int_or_none,
     mimetype2ext,
     parse_iso8601,
+    unified_timestamp,
     url_or_none,
 )
 
@@ -88,7 +89,7 @@ class AMPIE(InfoExtractor):
 
         self._sort_formats(formats)
 
-        timestamp = parse_iso8601(item.get('pubDate'), ' ') or parse_iso8601(item.get('dc-date'))
+        timestamp = unified_timestamp(item.get('pubDate'), ' ') or parse_iso8601(item.get('dc-date'))
 
         return {
             'id': video_id,
diff --git a/youtube_dl/extractor/bleacherreport.py b/youtube_dl/extractor/bleacherreport.py
index dc60224d0..d1bf8e829 100644
--- a/youtube_dl/extractor/bleacherreport.py
+++ b/youtube_dl/extractor/bleacherreport.py
@@ -90,13 +90,19 @@ class BleacherReportCMSIE(AMPIE):
     _VALID_URL = r'https?://(?:www\.)?bleacherreport\.com/video_embed\?id=(?P<id>[0-9a-f-]{36}|\d{5})'
     _TESTS = [{
         'url': 'http://bleacherreport.com/video_embed?id=8fd44c2f-3dc5-4821-9118-2c825a98c0e1&library=video-cms',
-        'md5': '2e4b0a997f9228ffa31fada5c53d1ed1',
+        'md5': '670b2d73f48549da032861130488c681',
         'info_dict': {
             'id': '8fd44c2f-3dc5-4821-9118-2c825a98c0e1',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': 'Cena vs. Rollins Would Expose the Heavyweight Division',
             'description': 'md5:984afb4ade2f9c0db35f3267ed88b36e',
+            'upload_date': '20150723',
+            'timestamp': 1437679032,
+
         },
+        'expected_warnings': [
+            'Unable to download f4m manifest'
+        ]
     }]
 
     def _real_extract(self, url):

From 11b68df7a4980f7f6175cdf2d7334fde11ff76b1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 27 Jan 2021 12:28:22 +0100
Subject: [PATCH 160/860] [abcnews] fix extraction(closes #12394)(closes
 #27920)

---
 youtube_dl/extractor/abcnews.py | 126 +++++++++++++++++---------------
 1 file changed, 67 insertions(+), 59 deletions(-)

diff --git a/youtube_dl/extractor/abcnews.py b/youtube_dl/extractor/abcnews.py
index 64ea6e6ed..908c83377 100644
--- a/youtube_dl/extractor/abcnews.py
+++ b/youtube_dl/extractor/abcnews.py
@@ -1,14 +1,15 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import calendar
 import re
-import time
 
 from .amp import AMPIE
 from .common import InfoExtractor
-from .youtube import YoutubeIE
-from ..compat import compat_urlparse
+from ..utils import (
+    parse_duration,
+    parse_iso8601,
+    try_get,
+)
 
 
 class AbcNewsVideoIE(AMPIE):
@@ -18,8 +19,8 @@ class AbcNewsVideoIE(AMPIE):
                         (?:
                             abcnews\.go\.com/
                             (?:
-                                [^/]+/video/(?P<display_id>[0-9a-z-]+)-|
-                                video/embed\?.*?\bid=
+                                (?:[^/]+/)*video/(?P<display_id>[0-9a-z-]+)-|
+                                video/(?:embed|itemfeed)\?.*?\bid=
                             )|
                             fivethirtyeight\.abcnews\.go\.com/video/embed/\d+/
                         )
@@ -49,6 +50,12 @@ class AbcNewsVideoIE(AMPIE):
     }, {
         'url': 'http://abcnews.go.com/2020/video/2020-husband-stands-teacher-jail-student-affairs-26119478',
         'only_matching': True,
+    }, {
+        'url': 'http://abcnews.go.com/video/itemfeed?id=46979033',
+        'only_matching': True,
+    }, {
+        'url': 'https://abcnews.go.com/GMA/News/video/history-christmas-story-67894761',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -69,28 +76,23 @@ class AbcNewsIE(InfoExtractor):
     _VALID_URL = r'https?://abcnews\.go\.com/(?:[^/]+/)+(?P<display_id>[0-9a-z-]+)/story\?id=(?P<id>\d+)'
 
     _TESTS = [{
-        'url': 'http://abcnews.go.com/Blotter/News/dramatic-video-rare-death-job-america/story?id=10498713#.UIhwosWHLjY',
+        # Youtube Embeds
+        'url': 'https://abcnews.go.com/Entertainment/peter-billingsley-child-actor-christmas-story-hollywood-power/story?id=51286501',
         'info_dict': {
-            'id': '10505354',
-            'ext': 'flv',
-            'display_id': 'dramatic-video-rare-death-job-america',
-            'title': 'Occupational Hazards',
-            'description': 'Nightline investigates the dangers that lurk at various jobs.',
-            'thumbnail': r're:^https?://.*\.jpg$',
-            'upload_date': '20100428',
-            'timestamp': 1272412800,
+            'id': '51286501',
+            'title': "Peter Billingsley: From child actor in 'A Christmas Story' to Hollywood power player",
+            'description': 'Billingsley went from a child actor to Hollywood power player.',
         },
-        'add_ie': ['AbcNewsVideo'],
+        'playlist_count': 5,
     }, {
         'url': 'http://abcnews.go.com/Entertainment/justin-timberlake-performs-stop-feeling-eurovision-2016/story?id=39125818',
         'info_dict': {
             'id': '38897857',
             'ext': 'mp4',
-            'display_id': 'justin-timberlake-performs-stop-feeling-eurovision-2016',
             'title': 'Justin Timberlake Drops Hints For Secret Single',
             'description': 'Lara Spencer reports the buzziest stories of the day in "GMA" Pop News.',
-            'upload_date': '20160515',
-            'timestamp': 1463329500,
+            'upload_date': '20160505',
+            'timestamp': 1462442280,
         },
         'params': {
             # m3u8 download
@@ -102,49 +104,55 @@ class AbcNewsIE(InfoExtractor):
     }, {
         'url': 'http://abcnews.go.com/Technology/exclusive-apple-ceo-tim-cook-iphone-cracking-software/story?id=37173343',
         'only_matching': True,
+    }, {
+        # inline.type == 'video'
+        'url': 'http://abcnews.go.com/Technology/exclusive-apple-ceo-tim-cook-iphone-cracking-software/story?id=37173343',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
-        video_id = mobj.group('id')
+        story_id = self._match_id(url)
+        webpage = self._download_webpage(url, story_id)
+        story = self._parse_json(self._search_regex(
+            r"window\['__abcnews__'\]\s*=\s*({.+?});",
+            webpage, 'data'), story_id)['page']['content']['story']['everscroll'][0]
+        article_contents = story.get('articleContents') or {}
 
-        webpage = self._download_webpage(url, video_id)
-        video_url = self._search_regex(
-            r'window\.abcnvideo\.url\s*=\s*"([^"]+)"', webpage, 'video URL')
-        full_video_url = compat_urlparse.urljoin(url, video_url)
+        def entries():
+            featured_video = story.get('featuredVideo') or {}
+            feed = try_get(featured_video, lambda x: x['video']['feed'])
+            if feed:
+                yield {
+                    '_type': 'url',
+                    'id': featured_video.get('id'),
+                    'title': featured_video.get('name'),
+                    'url': feed,
+                    'thumbnail': featured_video.get('images'),
+                    'description': featured_video.get('description'),
+                    'timestamp': parse_iso8601(featured_video.get('uploadDate')),
+                    'duration': parse_duration(featured_video.get('duration')),
+                    'ie_key': AbcNewsVideoIE.ie_key(),
+                }
 
-        youtube_url = YoutubeIE._extract_url(webpage)
+            for inline in (article_contents.get('inlines') or []):
+                inline_type = inline.get('type')
+                if inline_type == 'iframe':
+                    iframe_url = try_get(inline, lambda x: x['attrs']['src'])
+                    if iframe_url:
+                        yield self.url_result(iframe_url)
+                elif inline_type == 'video':
+                    video_id = inline.get('id')
+                    if video_id:
+                        yield {
+                            '_type': 'url',
+                            'id': video_id,
+                            'url': 'http://abcnews.go.com/video/embed?id=' + video_id,
+                            'thumbnail': inline.get('imgSrc') or inline.get('imgDefault'),
+                            'description': inline.get('description'),
+                            'duration': parse_duration(inline.get('duration')),
+                            'ie_key': AbcNewsVideoIE.ie_key(),
+                        }
 
-        timestamp = None
-        date_str = self._html_search_regex(
-            r'<span[^>]+class="timestamp">([^<]+)</span>',
-            webpage, 'timestamp', fatal=False)
-        if date_str:
-            tz_offset = 0
-            if date_str.endswith(' ET'):  # Eastern Time
-                tz_offset = -5
-                date_str = date_str[:-3]
-            date_formats = ['%b. %d, %Y', '%b %d, %Y, %I:%M %p']
-            for date_format in date_formats:
-                try:
-                    timestamp = calendar.timegm(time.strptime(date_str.strip(), date_format))
-                except ValueError:
-                    continue
-            if timestamp is not None:
-                timestamp -= tz_offset * 3600
-
-        entry = {
-            '_type': 'url_transparent',
-            'ie_key': AbcNewsVideoIE.ie_key(),
-            'url': full_video_url,
-            'id': video_id,
-            'display_id': display_id,
-            'timestamp': timestamp,
-        }
-
-        if youtube_url:
-            entries = [entry, self.url_result(youtube_url, ie=YoutubeIE.ie_key())]
-            return self.playlist_result(entries)
-
-        return entry
+        return self.playlist_result(
+            entries(), story_id, article_contents.get('headline'),
+            article_contents.get('subHead'))

From c669554ef5491302eb20fc2bcb52339ea1a4ac1a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 27 Jan 2021 14:51:30 +0100
Subject: [PATCH 161/860] [medialaan] add support DPG Media MyChannels based
 websites

closes #14871
closes #15597
closes #16106
closes #16489
---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/generic.py    |  23 +++
 youtube_dl/extractor/medialaan.py  | 303 +++++++----------------------
 youtube_dl/extractor/vtm.py        |  62 ++++++
 4 files changed, 160 insertions(+), 229 deletions(-)
 create mode 100644 youtube_dl/extractor/vtm.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index c554a8504..ab8d6a5a5 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1459,6 +1459,7 @@ from .vrv import (
     VRVSeriesIE,
 )
 from .vshare import VShareIE
+from .vtm import VTMIE
 from .medialaan import MedialaanIE
 from .vube import VubeIE
 from .vuclip import VuClipIE
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 780971a92..09e680c96 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -128,6 +128,7 @@ from .zype import ZypeIE
 from .odnoklassniki import OdnoklassnikiIE
 from .kinja import KinjaEmbedIE
 from .arcpublishing import ArcPublishingIE
+from .medialaan import MedialaanIE
 
 
 class GenericIE(InfoExtractor):
@@ -2223,6 +2224,20 @@ class GenericIE(InfoExtractor):
                 'duration': 1581,
             },
         },
+        {
+            # MyChannels SDK embed
+            # https://www.24kitchen.nl/populair/deskundige-dit-waarom-sommigen-gevoelig-zijn-voor-voedselallergieen
+            'url': 'https://www.demorgen.be/nieuws/burgemeester-rotterdam-richt-zich-in-videoboodschap-tot-relschoppers-voelt-het-goed~b0bcfd741/',
+            'md5': '90c0699c37006ef18e198c032d81739c',
+            'info_dict': {
+                'id': '194165',
+                'ext': 'mp4',
+                'title': 'Burgemeester Aboutaleb spreekt relschoppers toe',
+                'timestamp': 1611740340,
+                'upload_date': '20210127',
+                'duration': 159,
+            },
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -2462,6 +2477,9 @@ class GenericIE(InfoExtractor):
         webpage = self._webpage_read_content(
             full_response, url, video_id, prefix=first_bytes)
 
+        if '<title>DPG Media Privacy Gate</title>' in webpage:
+            webpage = self._download_webpage(url, video_id)
+
         self.report_extraction(video_id)
 
         # Is it an RSS feed, a SMIL file, an XSPF playlist or a MPD manifest?
@@ -2593,6 +2611,11 @@ class GenericIE(InfoExtractor):
         if arc_urls:
             return self.playlist_from_matches(arc_urls, video_id, video_title, ie=ArcPublishingIE.ie_key())
 
+        mychannels_urls = MedialaanIE._extract_urls(webpage)
+        if mychannels_urls:
+            return self.playlist_from_matches(
+                mychannels_urls, video_id, video_title, ie=MedialaanIE.ie_key())
+
         # Look for embedded rtl.nl player
         matches = re.findall(
             r'<iframe[^>]+?src="((?:https?:)?//(?:(?:www|static)\.)?rtl\.nl/(?:system/videoplayer/[^"]+(?:video_)?)?embed[^"]+)"',
diff --git a/youtube_dl/extractor/medialaan.py b/youtube_dl/extractor/medialaan.py
index 50d5db802..788acf7fb 100644
--- a/youtube_dl/extractor/medialaan.py
+++ b/youtube_dl/extractor/medialaan.py
@@ -2,268 +2,113 @@ from __future__ import unicode_literals
 
 import re
 
-from .gigya import GigyaBaseIE
-
-from ..compat import compat_str
+from .common import InfoExtractor
 from ..utils import (
+    extract_attributes,
     int_or_none,
-    parse_duration,
-    try_get,
-    unified_timestamp,
+    mimetype2ext,
+    parse_iso8601,
 )
 
 
-class MedialaanIE(GigyaBaseIE):
+class MedialaanIE(InfoExtractor):
     _VALID_URL = r'''(?x)
                     https?://
-                        (?:www\.|nieuws\.)?
                         (?:
-                            (?P<site_id>vtm|q2|vtmkzoom)\.be/
-                            (?:
-                                video(?:/[^/]+/id/|/?\?.*?\baid=)|
-                                (?:[^/]+/)*
-                            )
+                            (?:embed\.)?mychannels.video/embed/|
+                            embed\.mychannels\.video/(?:s(?:dk|cript)/)?production/|
+                            (?:www\.)?(?:
+                                (?:
+                                    7sur7|
+                                    demorgen|
+                                    hln|
+                                    joe|
+                                    qmusic
+                                )\.be|
+                                (?:
+                                    [abe]d|
+                                    bndestem|
+                                    destentor|
+                                    gelderlander|
+                                    pzc|
+                                    tubantia|
+                                    volkskrant
+                                )\.nl
+                            )/video/(?:[^/]+/)*[^/?&#]+~p
                         )
-                        (?P<id>[^/?#&]+)
+                        (?P<id>\d+)
                     '''
-    _NETRC_MACHINE = 'medialaan'
-    _APIKEY = '3_HZ0FtkMW_gOyKlqQzW5_0FHRC7Nd5XpXJZcDdXY4pk5eES2ZWmejRW5egwVm4ug-'
-    _SITE_TO_APP_ID = {
-        'vtm': 'vtm_watch',
-        'q2': 'q2',
-        'vtmkzoom': 'vtmkzoom',
-    }
     _TESTS = [{
-        # vod
-        'url': 'http://vtm.be/video/volledige-afleveringen/id/vtm_20170219_VM0678361_vtmwatch',
+        'url': 'https://www.bndestem.nl/video/de-terugkeer-van-ally-de-aap-en-wie-vertrekt-er-nog-bij-nac~p193993',
         'info_dict': {
-            'id': 'vtm_20170219_VM0678361_vtmwatch',
+            'id': '193993',
             'ext': 'mp4',
-            'title': 'Allemaal Chris afl. 6',
-            'description': 'md5:4be86427521e7b07e0adb0c9c554ddb2',
-            'timestamp': 1487533280,
-            'upload_date': '20170219',
-            'duration': 2562,
-            'series': 'Allemaal Chris',
-            'season': 'Allemaal Chris',
-            'season_number': 1,
-            'season_id': '256936078124527',
-            'episode': 'Allemaal Chris afl. 6',
-            'episode_number': 6,
-            'episode_id': '256936078591527',
+            'title': 'De terugkeer van Ally de Aap en wie vertrekt er nog bij NAC?',
+            'timestamp': 1611663540,
+            'upload_date': '20210126',
+            'duration': 238,
         },
         'params': {
             'skip_download': True,
         },
-        'skip': 'Requires account credentials',
     }, {
-        # clip
-        'url': 'http://vtm.be/video?aid=168332',
-        'info_dict': {
-            'id': '168332',
-            'ext': 'mp4',
-            'title': '"Veronique liegt!"',
-            'description': 'md5:1385e2b743923afe54ba4adc38476155',
-            'timestamp': 1489002029,
-            'upload_date': '20170308',
-            'duration': 96,
-        },
-    }, {
-        # vod
-        'url': 'http://vtm.be/video/volledige-afleveringen/id/257107153551000',
+        'url': 'https://www.gelderlander.nl/video/kanalen/degelderlander~c320/series/snel-nieuws~s984/noodbevel-in-doetinchem-politie-stuurt-mensen-centrum-uit~p194093',
         'only_matching': True,
     }, {
-        # vod
-        'url': 'http://vtm.be/video?aid=163157',
+        'url': 'https://embed.mychannels.video/sdk/production/193993?options=TFTFF_default',
         'only_matching': True,
     }, {
-        # vod
-        'url': 'http://www.q2.be/video/volledige-afleveringen/id/2be_20170301_VM0684442_q2',
+        'url': 'https://embed.mychannels.video/script/production/193993',
         'only_matching': True,
     }, {
-        # clip
-        'url': 'http://vtmkzoom.be/k3-dansstudio/een-nieuw-seizoen-van-k3-dansstudio',
+        'url': 'https://embed.mychannels.video/production/193993',
         'only_matching': True,
     }, {
-        # http/s redirect
-        'url': 'https://vtmkzoom.be/video?aid=45724',
-        'info_dict': {
-            'id': '257136373657000',
-            'ext': 'mp4',
-            'title': 'K3 Dansstudio Ushuaia afl.6',
-        },
-        'params': {
-            'skip_download': True,
-        },
-        'skip': 'Requires account credentials',
+        'url': 'https://mychannels.video/embed/193993',
+        'only_matching': True,
     }, {
-        # nieuws.vtm.be
-        'url': 'https://nieuws.vtm.be/stadion/stadion/genk-nog-moeilijk-programma',
+        'url': 'https://embed.mychannels.video/embed/193993',
         'only_matching': True,
     }]
 
-    def _real_initialize(self):
-        self._logged_in = False
-
-    def _login(self):
-        username, password = self._get_login_info()
-        if username is None:
-            self.raise_login_required()
-
-        auth_data = {
-            'APIKey': self._APIKEY,
-            'sdk': 'js_6.1',
-            'format': 'json',
-            'loginID': username,
-            'password': password,
-        }
-
-        auth_info = self._gigya_login(auth_data)
-
-        self._uid = auth_info['UID']
-        self._uid_signature = auth_info['UIDSignature']
-        self._signature_timestamp = auth_info['signatureTimestamp']
-
-        self._logged_in = True
+    @staticmethod
+    def _extract_urls(webpage):
+        entries = []
+        for element in re.findall(r'(<div[^>]+data-mychannels-type="video"[^>]*>)', webpage):
+            mychannels_id = extract_attributes(element).get('data-mychannels-id')
+            if mychannels_id:
+                entries.append('https://mychannels.video/embed/' + mychannels_id)
+        return entries
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id, site_id = mobj.group('id', 'site_id')
+        production_id = self._match_id(url)
+        production = self._download_json(
+            'https://embed.mychannels.video/sdk/production/' + production_id,
+            production_id, query={'options': 'UUUU_default'})['productions'][0]
+        title = production['title']
 
-        webpage = self._download_webpage(url, video_id)
-
-        config = self._parse_json(
-            self._search_regex(
-                r'videoJSConfig\s*=\s*JSON\.parse\(\'({.+?})\'\);',
-                webpage, 'config', default='{}'), video_id,
-            transform_source=lambda s: s.replace(
-                '\\\\', '\\').replace(r'\"', '"').replace(r"\'", "'"))
-
-        vod_id = config.get('vodId') or self._search_regex(
-            (r'\\"vodId\\"\s*:\s*\\"(.+?)\\"',
-             r'"vodId"\s*:\s*"(.+?)"',
-             r'<[^>]+id=["\']vod-(\d+)'),
-            webpage, 'video_id', default=None)
-
-        # clip, no authentication required
-        if not vod_id:
-            player = self._parse_json(
-                self._search_regex(
-                    r'vmmaplayer\(({.+?})\);', webpage, 'vmma player',
-                    default=''),
-                video_id, transform_source=lambda s: '[%s]' % s, fatal=False)
-            if player:
-                video = player[-1]
-                if video['videoUrl'] in ('http', 'https'):
-                    return self.url_result(video['url'], MedialaanIE.ie_key())
-                info = {
-                    'id': video_id,
-                    'url': video['videoUrl'],
-                    'title': video['title'],
-                    'thumbnail': video.get('imageUrl'),
-                    'timestamp': int_or_none(video.get('createdDate')),
-                    'duration': int_or_none(video.get('duration')),
-                }
+        formats = []
+        for source in (production.get('sources') or []):
+            src = source.get('src')
+            if not src:
+                continue
+            ext = mimetype2ext(source.get('type'))
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    src, production_id, 'mp4', 'm3u8_native',
+                    m3u8_id='hls', fatal=False))
             else:
-                info = self._parse_html5_media_entries(
-                    url, webpage, video_id, m3u8_id='hls')[0]
-                info.update({
-                    'id': video_id,
-                    'title': self._html_search_meta('description', webpage),
-                    'duration': parse_duration(self._html_search_meta('duration', webpage)),
+                formats.append({
+                    'ext': ext,
+                    'url': src,
                 })
-        # vod, authentication required
-        else:
-            if not self._logged_in:
-                self._login()
+        self._sort_formats(formats)
 
-            settings = self._parse_json(
-                self._search_regex(
-                    r'jQuery\.extend\(Drupal\.settings\s*,\s*({.+?})\);',
-                    webpage, 'drupal settings', default='{}'),
-                video_id)
-
-            def get(container, item):
-                return try_get(
-                    settings, lambda x: x[container][item],
-                    compat_str) or self._search_regex(
-                    r'"%s"\s*:\s*"([^"]+)' % item, webpage, item,
-                    default=None)
-
-            app_id = get('vod', 'app_id') or self._SITE_TO_APP_ID.get(site_id, 'vtm_watch')
-            sso = get('vod', 'gigyaDatabase') or 'vtm-sso'
-
-            data = self._download_json(
-                'http://vod.medialaan.io/api/1.0/item/%s/video' % vod_id,
-                video_id, query={
-                    'app_id': app_id,
-                    'user_network': sso,
-                    'UID': self._uid,
-                    'UIDSignature': self._uid_signature,
-                    'signatureTimestamp': self._signature_timestamp,
-                })
-
-            formats = self._extract_m3u8_formats(
-                data['response']['uri'], video_id, entry_protocol='m3u8_native',
-                ext='mp4', m3u8_id='hls')
-
-            self._sort_formats(formats)
-
-            info = {
-                'id': vod_id,
-                'formats': formats,
-            }
-
-            api_key = get('vod', 'apiKey')
-            channel = get('medialaanGigya', 'channel')
-
-            if api_key:
-                videos = self._download_json(
-                    'http://vod.medialaan.io/vod/v2/videos', video_id, fatal=False,
-                    query={
-                        'channels': channel,
-                        'ids': vod_id,
-                        'limit': 1,
-                        'apikey': api_key,
-                    })
-                if videos:
-                    video = try_get(
-                        videos, lambda x: x['response']['videos'][0], dict)
-                    if video:
-                        def get(container, item, expected_type=None):
-                            return try_get(
-                                video, lambda x: x[container][item], expected_type)
-
-                        def get_string(container, item):
-                            return get(container, item, compat_str)
-
-                        info.update({
-                            'series': get_string('program', 'title'),
-                            'season': get_string('season', 'title'),
-                            'season_number': int_or_none(get('season', 'number')),
-                            'season_id': get_string('season', 'id'),
-                            'episode': get_string('episode', 'title'),
-                            'episode_number': int_or_none(get('episode', 'number')),
-                            'episode_id': get_string('episode', 'id'),
-                            'duration': int_or_none(
-                                video.get('duration')) or int_or_none(
-                                video.get('durationMillis'), scale=1000),
-                            'title': get_string('episode', 'title'),
-                            'description': get_string('episode', 'text'),
-                            'timestamp': unified_timestamp(get_string(
-                                'publication', 'begin')),
-                        })
-
-            if not info.get('title'):
-                info['title'] = try_get(
-                    config, lambda x: x['videoConfig']['title'],
-                    compat_str) or self._html_search_regex(
-                    r'\\"title\\"\s*:\s*\\"(.+?)\\"', webpage, 'title',
-                    default=None) or self._og_search_title(webpage)
-
-        if not info.get('description'):
-            info['description'] = self._html_search_regex(
-                r'<div[^>]+class="field-item\s+even">\s*<p>(.+?)</p>',
-                webpage, 'description', default=None)
-
-        return info
+        return {
+            'id': production_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': production.get('posterUrl'),
+            'timestamp': parse_iso8601(production.get('publicationDate'), ' '),
+            'duration': int_or_none(production.get('duration')) or None,
+        }
diff --git a/youtube_dl/extractor/vtm.py b/youtube_dl/extractor/vtm.py
new file mode 100644
index 000000000..093f1aa69
--- /dev/null
+++ b/youtube_dl/extractor/vtm.py
@@ -0,0 +1,62 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    try_get,
+)
+
+
+class VTMIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?vtm\.be/([^/?&#]+)~v(?P<id>[0-9a-f]{8}(?:-[0-9a-f]{4}){3}-[0-9a-f]{12})'
+    _TEST = {
+        'url': 'https://vtm.be/gast-vernielt-genkse-hotelkamer~ve7534523-279f-4b4d-a5c9-a33ffdbe23e1',
+        'md5': '37dca85fbc3a33f2de28ceb834b071f8',
+        'info_dict': {
+            'id': '192445',
+            'ext': 'mp4',
+            'title': 'Gast vernielt Genkse hotelkamer',
+            'timestamp': 1611060180,
+            'upload_date': '20210119',
+            'duration': 74,
+            # TODO: fix url _type result processing
+            # 'series': 'Op Interventie',
+        }
+    }
+
+    def _real_extract(self, url):
+        uuid = self._match_id(url)
+        video = self._download_json(
+            'https://omc4vm23offuhaxx6hekxtzspi.appsync-api.eu-west-1.amazonaws.com/graphql',
+            uuid, query={
+                'query': '''{
+  getComponent(type: Video, uuid: "%s") {
+    ... on Video {
+      description
+      duration
+      myChannelsVideo
+      program {
+        title
+      }
+      publishedAt
+      title
+    }
+  }
+}''' % uuid,
+            }, headers={
+                'x-api-key': 'da2-lz2cab4tfnah3mve6wiye4n77e',
+            })['data']['getComponent']
+
+        return {
+            '_type': 'url',
+            'id': uuid,
+            'title': video.get('title'),
+            'url': 'http://mychannels.video/embed/%d' % video['myChannelsVideo'],
+            'description': video.get('description'),
+            'timestamp': parse_iso8601(video.get('publishedAt')),
+            'duration': int_or_none(video.get('duration')),
+            'series': try_get(video, lambda x: x['program']['title']),
+            'ie_key': 'Medialaan',
+        }

From fd95fc33b13d732002d53c35521f17184d14cc21 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adrian=20Heine=20n=C3=A9=20Lang?= <mail@adrianheine.de>
Date: Wed, 27 Jan 2021 20:06:12 +0100
Subject: [PATCH 162/860] [awaan] Extract uploader id (#27963)

---
 youtube_dl/extractor/awaan.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/awaan.py b/youtube_dl/extractor/awaan.py
index a2603bbff..3a7700cd4 100644
--- a/youtube_dl/extractor/awaan.py
+++ b/youtube_dl/extractor/awaan.py
@@ -48,6 +48,7 @@ class AWAANBaseIE(InfoExtractor):
             'duration': int_or_none(video_data.get('duration')),
             'timestamp': parse_iso8601(video_data.get('create_time'), ' '),
             'is_live': is_live,
+            'uploader_id': video_data.get('user_id'),
         }
 
 
@@ -107,6 +108,7 @@ class AWAANLiveIE(AWAANBaseIE):
             'title': 're:Dubai Al Oula [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
             'upload_date': '20150107',
             'timestamp': 1420588800,
+            'uploader_id': '71',
         },
         'params': {
             # m3u8 download

From 7b8fa658f88d53066f7a2ad00df19697552cf286 Mon Sep 17 00:00:00 2001
From: knapior <knapior93@gmail.com>
Date: Wed, 27 Jan 2021 20:43:20 +0100
Subject: [PATCH 163/860] [cda] Improve birth validation detection (closes
 #14022) (#27929)

Co-authored-by: Sergey M <dstftw@gmail.com>
---
 youtube_dl/extractor/cda.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cda.py b/youtube_dl/extractor/cda.py
index d67900e62..6429454fb 100644
--- a/youtube_dl/extractor/cda.py
+++ b/youtube_dl/extractor/cda.py
@@ -96,7 +96,7 @@ class CDAIE(InfoExtractor):
             raise ExtractorError('This video is only available for premium users.', expected=True)
 
         need_confirm_age = False
-        if self._html_search_regex(r'(<form[^>]+action="/a/validatebirth")',
+        if self._html_search_regex(r'(<form[^>]+action="[^"]*/a/validatebirth[^"]*")',
                                    webpage, 'birthday validate form', default=None):
             webpage = self._download_age_confirm_page(
                 url, video_id, note='Confirming age')

From 0b4f03a56394dee070f6e1723af8ce3a9ce44bfb Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 28 Jan 2021 16:22:24 +0100
Subject: [PATCH 164/860] [youtube] improve DASH formats file size extraction

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f57099f8c..b254ceced 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2370,7 +2370,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             mpd_url, video_id, fatal=dash_mpd_fatal,
                             formats_dict=self._formats):
                         if not df.get('filesize'):
-                            df['filesize'] = _extract_filesize(df['url'])
+                            df['filesize'] = _extract_filesize(df.get('fragment_base_url') or df['url'])
                         # Do not overwrite DASH format found in some previous DASH manifest
                         if df['format_id'] not in dash_formats:
                             dash_formats[df['format_id']] = df

From 56a7ee903360fe89cb5372805473b508153fb856 Mon Sep 17 00:00:00 2001
From: ping <ping@users.noreply.github.com>
Date: Fri, 29 Jan 2021 16:02:18 +0800
Subject: [PATCH 165/860] [vlive] Fix error message decoding for python 2
 (#28004)

---
 youtube_dl/extractor/vlive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index 6224e6200..e2f5d81b8 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -116,7 +116,7 @@ class VLiveIE(VLiveBaseIE):
                 headers={'Referer': 'https://www.vlive.tv/'}, query=query)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                self.raise_login_required(json.loads(e.cause.read().decode())['message'])
+                self.raise_login_required(json.loads(e.cause.read().decode('utf-8'))['message'])
             raise
 
     def _real_extract(self, url):

From ba15b2fee673f76c2cff2f193d5b4a19029b501f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 29 Jan 2021 11:52:48 +0100
Subject: [PATCH 166/860] [googledrive] report download page errors(closes
 #28005)

---
 youtube_dl/extractor/googledrive.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/googledrive.py b/youtube_dl/extractor/googledrive.py
index de8c80e36..3f2de00f1 100644
--- a/youtube_dl/extractor/googledrive.py
+++ b/youtube_dl/extractor/googledrive.py
@@ -7,6 +7,7 @@ from ..compat import compat_parse_qs
 from ..utils import (
     determine_ext,
     ExtractorError,
+    get_element_by_class,
     int_or_none,
     lowercase_escape,
     try_get,
@@ -237,7 +238,7 @@ class GoogleDriveIE(InfoExtractor):
                 if confirmation_webpage:
                     confirm = self._search_regex(
                         r'confirm=([^&"\']+)', confirmation_webpage,
-                        'confirmation code', fatal=False)
+                        'confirmation code', default=None)
                     if confirm:
                         confirmed_source_url = update_url_query(source_url, {
                             'confirm': confirm,
@@ -245,6 +246,11 @@ class GoogleDriveIE(InfoExtractor):
                         urlh = request_source_file(confirmed_source_url, 'confirmed source')
                         if urlh and urlh.headers.get('Content-Disposition'):
                             add_source_format(urlh)
+                    else:
+                        self.report_warning(
+                            get_element_by_class('uc-error-subcaption', confirmation_webpage)
+                            or get_element_by_class('uc-error-caption', confirmation_webpage)
+                            or 'unable to extract confirmation code')
 
         if not formats and reason:
             raise ExtractorError(reason, expected=True)

From a800838f5afad43a76a53bdb5f4c1b20c80ff202 Mon Sep 17 00:00:00 2001
From: nixxo <nixxo@protonmail.com>
Date: Fri, 29 Jan 2021 14:24:28 +0100
Subject: [PATCH 167/860] [vvvvid] add support for youtube embeds (#27825)

---
 youtube_dl/extractor/vvvvid.py | 35 ++++++++++++++++++++++++++++++----
 1 file changed, 31 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/vvvvid.py b/youtube_dl/extractor/vvvvid.py
index f4cae7fe9..778ce8b76 100644
--- a/youtube_dl/extractor/vvvvid.py
+++ b/youtube_dl/extractor/vvvvid.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from .youtube import YoutubeIE
 from ..utils import (
     ExtractorError,
     int_or_none,
@@ -47,6 +48,22 @@ class VVVVIDIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # video_type == 'video/youtube'
+        'url': 'https://www.vvvvid.it/show/404/one-punch-man/406/486683/trailer',
+        'md5': '33e0edfba720ad73a8782157fdebc648',
+        'info_dict': {
+            'id': 'RzmFKUDOUgw',
+            'ext': 'mp4',
+            'title': 'Trailer',
+            'upload_date': '20150906',
+            'description': 'md5:a5e802558d35247fee285875328c0b80',
+            'uploader_id': 'BandaiVisual',
+            'uploader': 'BANDAI NAMCO Arts Channel',
+        },
+        'params': {
+            'skip_download': True,
+        },
     }, {
         'url': 'https://www.vvvvid.it/show/434/perche-dovrei-guardarlo-di-dario-moccia/437/489048',
         'only_matching': True
@@ -154,12 +171,13 @@ class VVVVIDIE(InfoExtractor):
                     if season_number:
                         info['season_number'] = int(season_number)
 
-        for quality in ('_sd', ''):
+        video_type = video_data.get('video_type')
+        is_youtube = False
+        for quality in ('', '_sd'):
             embed_code = video_data.get('embed_info' + quality)
             if not embed_code:
                 continue
             embed_code = ds(embed_code)
-            video_type = video_data.get('video_type')
             if video_type in ('video/rcs', 'video/kenc'):
                 if video_type == 'video/kenc':
                     kenc = self._download_json(
@@ -172,19 +190,28 @@ class VVVVIDIE(InfoExtractor):
                     if kenc_message:
                         embed_code += '?' + ds(kenc_message)
                 formats.extend(self._extract_akamai_formats(embed_code, video_id))
+            elif video_type == 'video/youtube':
+                info.update({
+                    '_type': 'url_transparent',
+                    'ie_key': YoutubeIE.ie_key(),
+                    'url': embed_code,
+                })
+                is_youtube = True
+                break
             else:
                 formats.extend(self._extract_wowza_formats(
                     'http://sb.top-ix.org/videomg/_definst_/mp4:%s/playlist.m3u8' % embed_code, video_id))
             metadata_from_url(embed_code)
 
-        self._sort_formats(formats)
+        if not is_youtube:
+            self._sort_formats(formats)
+            info['formats'] = formats
 
         metadata_from_url(video_data.get('thumbnail'))
         info.update(self._extract_common_video_info(video_data))
         info.update({
             'id': video_id,
             'title': title,
-            'formats': formats,
             'duration': int_or_none(video_data.get('length')),
             'series': video_data.get('show_title'),
             'season_id': season_id,

From 8bf9591a70757c624a8ea5bf686040ed752246e0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adrian=20Heine=20n=C3=A9=20Lang?= <mail@adrianheine.de>
Date: Sat, 30 Jan 2021 13:44:01 +0100
Subject: [PATCH 168/860] [AENetworks] update AENetworksShowIE test playlist id
 (#27851)

---
 youtube_dl/extractor/aenetworks.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index a5d88ebbe..e55c03fd7 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -252,7 +252,7 @@ class AENetworksShowIE(AENetworksListBaseIE):
     _TESTS = [{
         'url': 'http://www.history.com/shows/ancient-aliens',
         'info_dict': {
-            'id': 'SH012427480000',
+            'id': 'SERIES1574',
             'title': 'Ancient Aliens',
             'description': 'md5:3f6d74daf2672ff3ae29ed732e37ea7f',
         },

From 67299f23d8b1894120e875edf97440de87e22308 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 1 Feb 2021 14:30:59 +0100
Subject: [PATCH 169/860] [youtube] Rewrite Extractor

- improve format sorting
- remove unused code(swf parsing, ...)
- fix series metadata extraction
- fix trailer video extraction
- improve error reporting
- extract video location
---
 youtube_dl/extractor/common.py  |   17 +-
 youtube_dl/extractor/youtube.py | 1717 +++++++++----------------------
 2 files changed, 483 insertions(+), 1251 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d5faa0eb7..8eb110f4e 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2064,7 +2064,7 @@ class InfoExtractor(object):
             })
         return entries
 
-    def _extract_mpd_formats(self, mpd_url, video_id, mpd_id=None, note=None, errnote=None, fatal=True, formats_dict={}, data=None, headers={}, query={}):
+    def _extract_mpd_formats(self, mpd_url, video_id, mpd_id=None, note=None, errnote=None, fatal=True, data=None, headers={}, query={}):
         res = self._download_xml_handle(
             mpd_url, video_id,
             note=note or 'Downloading MPD manifest',
@@ -2078,10 +2078,9 @@ class InfoExtractor(object):
         mpd_base_url = base_url(urlh.geturl())
 
         return self._parse_mpd_formats(
-            mpd_doc, mpd_id=mpd_id, mpd_base_url=mpd_base_url,
-            formats_dict=formats_dict, mpd_url=mpd_url)
+            mpd_doc, mpd_id, mpd_base_url, mpd_url)
 
-    def _parse_mpd_formats(self, mpd_doc, mpd_id=None, mpd_base_url='', formats_dict={}, mpd_url=None):
+    def _parse_mpd_formats(self, mpd_doc, mpd_id=None, mpd_base_url='', mpd_url=None):
         """
         Parse formats from MPD manifest.
         References:
@@ -2359,15 +2358,7 @@ class InfoExtractor(object):
                         else:
                             # Assuming direct URL to unfragmented media.
                             f['url'] = base_url
-
-                        # According to [1, 5.3.5.2, Table 7, page 35] @id of Representation
-                        # is not necessarily unique within a Period thus formats with
-                        # the same `format_id` are quite possible. There are numerous examples
-                        # of such manifests (see https://github.com/ytdl-org/youtube-dl/issues/15111,
-                        # https://github.com/ytdl-org/youtube-dl/issues/13919)
-                        full_info = formats_dict.get(representation_id, {}).copy()
-                        full_info.update(f)
-                        formats.append(full_info)
+                        formats.append(f)
                     else:
                         self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
         return formats
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b254ceced..5f6769878 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2,41 +2,34 @@
 
 from __future__ import unicode_literals
 
-
 import itertools
 import json
 import os.path
 import random
 import re
-import time
 import traceback
 
 from .common import InfoExtractor, SearchInfoExtractor
-from ..jsinterp import JSInterpreter
-from ..swfinterp import SWFInterpreter
 from ..compat import (
     compat_chr,
     compat_HTTPError,
     compat_parse_qs,
-    compat_urllib_parse_unquote,
+    compat_str,
     compat_urllib_parse_unquote_plus,
     compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse,
     compat_urlparse,
-    compat_str,
 )
+from ..jsinterp import JSInterpreter
 from ..utils import (
-    bool_or_none,
-    clean_html,
-    error_to_compat_str,
     ExtractorError,
+    clean_html,
     float_or_none,
-    get_element_by_id,
     int_or_none,
     mimetype2ext,
     parse_codecs,
     parse_duration,
-    remove_quotes,
+    qualities,
     remove_start,
     smuggle_url,
     str_or_none,
@@ -46,7 +39,6 @@ from ..utils import (
     unified_strdate,
     unsmuggle_url,
     update_url_query,
-    uppercase_escape,
     url_or_none,
     urlencode_postdata,
     urljoin,
@@ -68,12 +60,6 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM)'
 
-    def _set_language(self):
-        self._set_cookie(
-            '.youtube.com', 'PREF', 'f1=50000000&f6=8&hl=en',
-            # YouTube sets the expire time to about two months
-            expire_time=time.time() + 2 * 30 * 24 * 3600)
-
     def _ids_to_results(self, ids):
         return [
             self.url_result(vid_id, 'Youtube', video_id=vid_id)
@@ -265,7 +251,6 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     def _real_initialize(self):
         if self._downloader is None:
             return
-        self._set_language()
         if not self._login():
             return
 
@@ -282,19 +267,17 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
     _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
 
-    def _call_api(self, ep, query, video_id):
+    def _call_api(self, ep, query, video_id, fatal=True):
         data = self._DEFAULT_API_DATA.copy()
         data.update(query)
 
-        response = self._download_json(
+        return self._download_json(
             'https://www.youtube.com/youtubei/v1/%s' % ep, video_id=video_id,
             note='Downloading API JSON', errnote='Unable to download API page',
-            data=json.dumps(data).encode('utf8'),
+            data=json.dumps(data).encode('utf8'), fatal=fatal,
             headers={'content-type': 'application/json'},
             query={'key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'})
 
-        return response
-
     def _extract_yt_initial_data(self, video_id, webpage):
         return self._parse_json(
             self._search_regex(
@@ -414,116 +397,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                      )
                      (?(1).+)?                                                # if we found the ID, everything can follow
                      $""" % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
-    _NEXT_URL_RE = r'[\?&]next_url=([^&]+)'
     _PLAYER_INFO_RE = (
-        r'/(?P<id>[a-zA-Z0-9_-]{8,})/player_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?/base\.(?P<ext>[a-z]+)$',
-        r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.(?P<ext>[a-z]+)$',
+        r'/(?P<id>[a-zA-Z0-9_-]{8,})/player_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?/base\.js$',
+        r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
     )
-    _formats = {
-        '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
-        '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
-        '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
-        '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
-        '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
-        '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
-        '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
-        '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
-        # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
-        '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
-        '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
-        '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
-        '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
-        '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
-        '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
-        '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
-        '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
-        '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
-
-
-        # 3D videos
-        '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
-        '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
-        '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
-        '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
-        '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
-        '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
-        '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
-
-        # Apple HTTP Live Streaming
-        '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
-        '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
-        '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
-        '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
-        '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
-        '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
-        '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
-        '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
-
-        # DASH mp4 video
-        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
-        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
-        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
-        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
-        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
-        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
-        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
-        '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
-        '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
-        '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
-        '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
-        '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
-
-        # Dash mp4 audio
-        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
-        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
-        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
-        '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
-        '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
-        '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
-        '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
-
-        # Dash webm
-        '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
-        '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
-        '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
-        '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
-        '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
-        '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
-        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
-        '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
-        '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
-        '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
-        '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
-        '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
-        '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
-        '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
-        '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
-        # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
-        '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
-        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
-        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
-        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
-        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
-        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
-
-        # Dash webm audio
-        '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
-        '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
-
-        # Dash webm audio with opus inside
-        '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
-        '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
-        '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
-
-        # RTMP (unnamed)
-        '_rtmp': {'protocol': 'rtmp'},
-
-        # av01 video only formats sometimes served with "unknown" codecs
-        '394': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
-        '395': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
-        '396': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
-        '397': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
-    }
     _SUBTITLE_FORMATS = ('srv1', 'srv2', 'srv3', 'ttml', 'vtt')
 
     _GEO_BYPASS = False
@@ -566,7 +443,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_id': 'setindia',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/setindia',
                 'age_limit': 18,
-            }
+            },
+            'skip': 'Private video',
         },
         {
             'url': 'https://www.youtube.com/watch?v=BaW_jenozKc&v=yZIXLfi8CZQ',
@@ -640,7 +518,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_id': 'TheAmazingAtheist',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheAmazingAtheist',
                 'title': 'Burning Everyone\'s Koran',
-                'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms\n\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
+                'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms \r\n\r\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
             }
         },
         # Normal age-gate video (No vevo, embed allowed), available via embed page
@@ -676,11 +554,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'upload_date': '20100430',
                 'uploader_id': 'deadmau5',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
-                'creator': 'Dada Life, deadmau5',
-                'description': 'md5:12c56784b8032162bb936a5f76d55360',
+                'creator': 'deadmau5',
+                'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
                 'uploader': 'deadmau5',
                 'title': 'Deadmau5 - Some Chords (HD)',
-                'alt_title': 'This Machine Kills Some Chords',
+                'alt_title': 'Some Chords',
             },
             'expected_warnings': [
                 'DASH manifest missing',
@@ -775,69 +653,64 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         },
         {
             # Multifeed videos (multiple cameras), URL is for Main Camera
-            'url': 'https://www.youtube.com/watch?v=jqWvoWXjCVs',
+            'url': 'https://www.youtube.com/watch?v=jvGDaLqkpTg',
             'info_dict': {
-                'id': 'jqWvoWXjCVs',
-                'title': 'teamPGP: Rocket League Noob Stream',
-                'description': 'md5:dc7872fb300e143831327f1bae3af010',
+                'id': 'jvGDaLqkpTg',
+                'title': 'Tom Clancy Free Weekend Rainbow Whatever',
+                'description': 'md5:e03b909557865076822aa169218d6a5d',
             },
             'playlist': [{
                 'info_dict': {
-                    'id': 'jqWvoWXjCVs',
+                    'id': 'jvGDaLqkpTg',
                     'ext': 'mp4',
-                    'title': 'teamPGP: Rocket League Noob Stream (Main Camera)',
-                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
-                    'duration': 7335,
-                    'upload_date': '20150721',
-                    'uploader': 'Beer Games Beer',
-                    'uploader_id': 'beergamesbeer',
-                    'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
-                    'license': 'Standard YouTube License',
+                    'title': 'Tom Clancy Free Weekend Rainbow Whatever (Main Camera)',
+                    'description': 'md5:e03b909557865076822aa169218d6a5d',
+                    'duration': 10643,
+                    'upload_date': '20161111',
+                    'uploader': 'Team PGP',
+                    'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
+                    'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
                 },
             }, {
                 'info_dict': {
-                    'id': '6h8e8xoXJzg',
+                    'id': '3AKt1R1aDnw',
                     'ext': 'mp4',
-                    'title': 'teamPGP: Rocket League Noob Stream (kreestuh)',
-                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
-                    'duration': 7337,
-                    'upload_date': '20150721',
-                    'uploader': 'Beer Games Beer',
-                    'uploader_id': 'beergamesbeer',
-                    'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
-                    'license': 'Standard YouTube License',
+                    'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 2)',
+                    'description': 'md5:e03b909557865076822aa169218d6a5d',
+                    'duration': 10991,
+                    'upload_date': '20161111',
+                    'uploader': 'Team PGP',
+                    'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
+                    'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
                 },
             }, {
                 'info_dict': {
-                    'id': 'PUOgX5z9xZw',
+                    'id': 'RtAMM00gpVc',
                     'ext': 'mp4',
-                    'title': 'teamPGP: Rocket League Noob Stream (grizzle)',
-                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
-                    'duration': 7337,
-                    'upload_date': '20150721',
-                    'uploader': 'Beer Games Beer',
-                    'uploader_id': 'beergamesbeer',
-                    'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
-                    'license': 'Standard YouTube License',
+                    'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 3)',
+                    'description': 'md5:e03b909557865076822aa169218d6a5d',
+                    'duration': 10995,
+                    'upload_date': '20161111',
+                    'uploader': 'Team PGP',
+                    'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
+                    'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
                 },
             }, {
                 'info_dict': {
-                    'id': 'teuwxikvS5k',
+                    'id': '6N2fdlP3C5U',
                     'ext': 'mp4',
-                    'title': 'teamPGP: Rocket League Noob Stream (zim)',
-                    'description': 'md5:dc7872fb300e143831327f1bae3af010',
-                    'duration': 7334,
-                    'upload_date': '20150721',
-                    'uploader': 'Beer Games Beer',
-                    'uploader_id': 'beergamesbeer',
-                    'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/beergamesbeer',
-                    'license': 'Standard YouTube License',
+                    'title': 'Tom Clancy Free Weekend Rainbow Whatever (Camera 4)',
+                    'description': 'md5:e03b909557865076822aa169218d6a5d',
+                    'duration': 10990,
+                    'upload_date': '20161111',
+                    'uploader': 'Team PGP',
+                    'uploader_id': 'UChORY56LMMETTuGjXaJXvLg',
+                    'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UChORY56LMMETTuGjXaJXvLg',
                 },
             }],
             'params': {
                 'skip_download': True,
             },
-            'skip': 'This video is not available.',
         },
         {
             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
@@ -931,7 +804,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': 'eQcmzGIKrzg',
                 'ext': 'mp4',
                 'title': 'Democratic Socialism and Foreign Policy | Bernie Sanders',
-                'description': 'md5:dda0d780d5a6e120758d1711d062a867',
+                'description': 'md5:13a2503d7b5904ef4b223aa101628f39',
                 'duration': 4060,
                 'upload_date': '20151119',
                 'uploader': 'Bernie Sanders',
@@ -978,7 +851,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': 'iqKdEhx-dD4',
                 'ext': 'mp4',
                 'title': 'Isolation - Mind Field (Ep 1)',
-                'description': 'md5:46a29be4ceffa65b92d277b93f463c0f',
+                'description': 'md5:f540112edec5d09fc8cc752d3d4ba3cd',
                 'duration': 2085,
                 'upload_date': '20170118',
                 'uploader': 'Vsauce',
@@ -1013,6 +886,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'params': {
                 'skip_download': True,
             },
+            'skip': 'This video has been removed for violating YouTube\'s policy on hate speech.',
         },
         {
             # itag 212
@@ -1098,6 +972,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'params': {
                 'skip_download': True,
             },
+            'skip': 'Video unavailable',
         },
         {
             # empty description results in an empty string
@@ -1147,24 +1022,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     def __init__(self, *args, **kwargs):
         super(YoutubeIE, self).__init__(*args, **kwargs)
+        self._code_cache = {}
         self._player_cache = {}
 
-    def report_video_info_webpage_download(self, video_id):
-        """Report attempt to download video info webpage."""
-        self.to_screen('%s: Downloading video info webpage' % video_id)
-
-    def report_information_extraction(self, video_id):
-        """Report attempt to extract video information."""
-        self.to_screen('%s: Extracting video information' % video_id)
-
-    def report_unavailable_format(self, video_id, format):
-        """Report extracted video URL."""
-        self.to_screen('%s: Format %s not available' % (video_id, format))
-
-    def report_rtmp_download(self):
-        """Indicate the download will use the RTMP protocol."""
-        self.to_screen('RTMP download detected')
-
     def _signature_cache_id(self, example_sig):
         """ Return a string representation of a signature """
         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
@@ -1177,40 +1037,27 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 break
         else:
             raise ExtractorError('Cannot identify player %r' % player_url)
-        return id_m.group('ext'), id_m.group('id')
+        return id_m.group('id')
 
     def _extract_signature_function(self, video_id, player_url, example_sig):
-        player_type, player_id = self._extract_player_info(player_url)
+        player_id = self._extract_player_info(player_url)
 
         # Read from filesystem cache
-        func_id = '%s_%s_%s' % (
-            player_type, player_id, self._signature_cache_id(example_sig))
+        func_id = 'js_%s_%s' % (
+            player_id, self._signature_cache_id(example_sig))
         assert os.path.basename(func_id) == func_id
 
         cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
         if cache_spec is not None:
             return lambda s: ''.join(s[i] for i in cache_spec)
 
-        download_note = (
-            'Downloading player %s' % player_url
-            if self._downloader.params.get('verbose') else
-            'Downloading %s player %s' % (player_type, player_id)
-        )
-        if player_type == 'js':
-            code = self._download_webpage(
+        if player_id not in self._code_cache:
+            self._code_cache[player_id] = self._download_webpage(
                 player_url, video_id,
-                note=download_note,
+                note='Downloading player ' + player_id,
                 errnote='Download of %s failed' % player_url)
-            res = self._parse_sig_js(code)
-        elif player_type == 'swf':
-            urlh = self._request_webpage(
-                player_url, video_id,
-                note=download_note,
-                errnote='Download of %s failed' % player_url)
-            code = urlh.read()
-            res = self._parse_sig_swf(code)
-        else:
-            assert False, 'Invalid player type %r' % player_type
+        code = self._code_cache[player_id]
+        res = self._parse_sig_js(code)
 
         test_string = ''.join(map(compat_chr, range(len(example_sig))))
         cache_res = res(test_string)
@@ -1279,14 +1126,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         initial_function = jsi.extract_function(funcname)
         return lambda s: initial_function([s])
 
-    def _parse_sig_swf(self, file_contents):
-        swfi = SWFInterpreter(file_contents)
-        TARGET_CLASSNAME = 'SignatureDecipher'
-        searched_class = swfi.extract_class(TARGET_CLASSNAME)
-        initial_function = swfi.extract_function(searched_class, 'decipher')
-        return lambda s: initial_function([s])
-
-    def _decrypt_signature(self, s, video_id, player_url, age_gate=False):
+    def _decrypt_signature(self, s, video_id, player_url):
         """Turn the encrypted s field into a working signature"""
 
         if player_url is None:
@@ -1313,158 +1153,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             raise ExtractorError(
                 'Signature extraction failed: ' + tb, cause=e)
 
-    def _get_subtitles(self, video_id, webpage):
-        try:
-            subs_doc = self._download_xml(
-                'https://video.google.com/timedtext?hl=en&type=list&v=%s' % video_id,
-                video_id, note=False)
-        except ExtractorError as err:
-            self._downloader.report_warning('unable to download video subtitles: %s' % error_to_compat_str(err))
-            return {}
-
-        sub_lang_list = {}
-        for track in subs_doc.findall('track'):
-            lang = track.attrib['lang_code']
-            if lang in sub_lang_list:
-                continue
-            sub_formats = []
-            for ext in self._SUBTITLE_FORMATS:
-                params = compat_urllib_parse_urlencode({
-                    'lang': lang,
-                    'v': video_id,
-                    'fmt': ext,
-                    'name': track.attrib['name'].encode('utf-8'),
-                })
-                sub_formats.append({
-                    'url': 'https://www.youtube.com/api/timedtext?' + params,
-                    'ext': ext,
-                })
-            sub_lang_list[lang] = sub_formats
-        if not sub_lang_list:
-            self._downloader.report_warning('video doesn\'t have subtitles')
-            return {}
-        return sub_lang_list
-
-    def _get_ytplayer_config(self, video_id, webpage):
-        patterns = (
-            # User data may contain arbitrary character sequences that may affect
-            # JSON extraction with regex, e.g. when '};' is contained the second
-            # regex won't capture the whole JSON. Yet working around by trying more
-            # concrete regex first keeping in mind proper quoted string handling
-            # to be implemented in future that will replace this workaround (see
-            # https://github.com/ytdl-org/youtube-dl/issues/7468,
-            # https://github.com/ytdl-org/youtube-dl/pull/7599)
-            r';ytplayer\.config\s*=\s*({.+?});ytplayer',
-            r';ytplayer\.config\s*=\s*({.+?});',
-        )
-        config = self._search_regex(
-            patterns, webpage, 'ytplayer.config', default=None)
-        if config:
-            return self._parse_json(
-                uppercase_escape(config), video_id, fatal=False)
-
-    def _get_automatic_captions(self, video_id, player_response, player_config):
-        """We need the webpage for getting the captions url, pass it as an
-           argument to speed up the process."""
-        self.to_screen('%s: Looking for automatic captions' % video_id)
-        err_msg = 'Couldn\'t find automatic captions for %s' % video_id
-        if not (player_response or player_config):
-            self._downloader.report_warning(err_msg)
-            return {}
-        try:
-            args = player_config.get('args') if player_config else {}
-            caption_url = args.get('ttsurl')
-            if caption_url:
-                timestamp = args['timestamp']
-                # We get the available subtitles
-                list_params = compat_urllib_parse_urlencode({
-                    'type': 'list',
-                    'tlangs': 1,
-                    'asrs': 1,
-                })
-                list_url = caption_url + '&' + list_params
-                caption_list = self._download_xml(list_url, video_id)
-                original_lang_node = caption_list.find('track')
-                if original_lang_node is None:
-                    self._downloader.report_warning('Video doesn\'t have automatic captions')
-                    return {}
-                original_lang = original_lang_node.attrib['lang_code']
-                caption_kind = original_lang_node.attrib.get('kind', '')
-
-                sub_lang_list = {}
-                for lang_node in caption_list.findall('target'):
-                    sub_lang = lang_node.attrib['lang_code']
-                    sub_formats = []
-                    for ext in self._SUBTITLE_FORMATS:
-                        params = compat_urllib_parse_urlencode({
-                            'lang': original_lang,
-                            'tlang': sub_lang,
-                            'fmt': ext,
-                            'ts': timestamp,
-                            'kind': caption_kind,
-                        })
-                        sub_formats.append({
-                            'url': caption_url + '&' + params,
-                            'ext': ext,
-                        })
-                    sub_lang_list[sub_lang] = sub_formats
-                return sub_lang_list
-
-            def make_captions(sub_url, sub_langs):
-                parsed_sub_url = compat_urllib_parse_urlparse(sub_url)
-                caption_qs = compat_parse_qs(parsed_sub_url.query)
-                captions = {}
-                for sub_lang in sub_langs:
-                    sub_formats = []
-                    for ext in self._SUBTITLE_FORMATS:
-                        caption_qs.update({
-                            'tlang': [sub_lang],
-                            'fmt': [ext],
-                        })
-                        sub_url = compat_urlparse.urlunparse(parsed_sub_url._replace(
-                            query=compat_urllib_parse_urlencode(caption_qs, True)))
-                        sub_formats.append({
-                            'url': sub_url,
-                            'ext': ext,
-                        })
-                    captions[sub_lang] = sub_formats
-                return captions
-
-            # New captions format as of 22.06.2017
-            if player_response:
-                renderer = player_response['captions']['playerCaptionsTracklistRenderer']
-                base_url = renderer['captionTracks'][0]['baseUrl']
-                sub_lang_list = []
-                for lang in renderer['translationLanguages']:
-                    lang_code = lang.get('languageCode')
-                    if lang_code:
-                        sub_lang_list.append(lang_code)
-                return make_captions(base_url, sub_lang_list)
-
-            # Some videos don't provide ttsurl but rather caption_tracks and
-            # caption_translation_languages (e.g. 20LmZk1hakA)
-            # Does not used anymore as of 22.06.2017
-            caption_tracks = args['caption_tracks']
-            caption_translation_languages = args['caption_translation_languages']
-            caption_url = compat_parse_qs(caption_tracks.split(',')[0])['u'][0]
-            sub_lang_list = []
-            for lang in caption_translation_languages.split(','):
-                lang_qs = compat_parse_qs(compat_urllib_parse_unquote_plus(lang))
-                sub_lang = lang_qs.get('lc', [None])[0]
-                if sub_lang:
-                    sub_lang_list.append(sub_lang)
-            return make_captions(caption_url, sub_lang_list)
-        # An extractor error can be raise by the download process if there are
-        # no automatic captions but there are subtitles
-        except (KeyError, IndexError, ExtractorError):
-            self._downloader.report_warning(err_msg)
-            return {}
-
-    def _mark_watched(self, video_id, video_info, player_response):
+    def _mark_watched(self, video_id, player_response):
         playback_url = url_or_none(try_get(
             player_response,
-            lambda x: x['playbackTracking']['videostatsPlaybackUrl']['baseUrl']) or try_get(
-            video_info, lambda x: x['videostats_playback_base_url'][0]))
+            lambda x: x['playbackTracking']['videostatsPlaybackUrl']['baseUrl']))
         if not playback_url:
             return
         parsed_playback_url = compat_urlparse.urlparse(playback_url)
@@ -1531,289 +1223,74 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         video_id = mobj.group(2)
         return video_id
 
-    def _extract_chapters_from_json(self, webpage, video_id, duration):
-        if not webpage:
-            return
-        data = self._extract_yt_initial_data(video_id, webpage)
-        if not data or not isinstance(data, dict):
-            return
-        chapters_list = try_get(
-            data,
-            lambda x: x['playerOverlays']
-                       ['playerOverlayRenderer']
-                       ['decoratedPlayerBarRenderer']
-                       ['decoratedPlayerBarRenderer']
-                       ['playerBar']
-                       ['chapteredPlayerBarRenderer']
-                       ['chapters'],
-            list)
-        if not chapters_list:
-            return
-
-        def chapter_time(chapter):
-            return float_or_none(
-                try_get(
-                    chapter,
-                    lambda x: x['chapterRenderer']['timeRangeStartMillis'],
-                    int),
-                scale=1000)
-        chapters = []
-        for next_num, chapter in enumerate(chapters_list, start=1):
-            start_time = chapter_time(chapter)
-            if start_time is None:
-                continue
-            end_time = (chapter_time(chapters_list[next_num])
-                        if next_num < len(chapters_list) else duration)
-            if end_time is None:
-                continue
-            title = try_get(
-                chapter, lambda x: x['chapterRenderer']['title']['simpleText'],
-                compat_str)
-            chapters.append({
-                'start_time': start_time,
-                'end_time': end_time,
-                'title': title,
-            })
-        return chapters
-
-    @staticmethod
-    def _extract_chapters_from_description(description, duration):
-        if not description:
-            return None
-        chapter_lines = re.findall(
-            r'(?:^|<br\s*/>)([^<]*<a[^>]+onclick=["\']yt\.www\.watch\.player\.seekTo[^>]+>(\d{1,2}:\d{1,2}(?::\d{1,2})?)</a>[^>]*)(?=$|<br\s*/>)',
-            description)
-        if not chapter_lines:
-            return None
-        chapters = []
-        for next_num, (chapter_line, time_point) in enumerate(
-                chapter_lines, start=1):
-            start_time = parse_duration(time_point)
-            if start_time is None:
-                continue
-            if start_time > duration:
-                break
-            end_time = (duration if next_num == len(chapter_lines)
-                        else parse_duration(chapter_lines[next_num][1]))
-            if end_time is None:
-                continue
-            if end_time > duration:
-                end_time = duration
-            if start_time > end_time:
-                break
-            chapter_title = re.sub(
-                r'<a[^>]+>[^<]+</a>', '', chapter_line).strip(' \t-')
-            chapter_title = re.sub(r'\s+', ' ', chapter_title)
-            chapters.append({
-                'start_time': start_time,
-                'end_time': end_time,
-                'title': chapter_title,
-            })
-        return chapters
-
-    def _extract_chapters(self, webpage, description, video_id, duration):
-        return (self._extract_chapters_from_json(webpage, video_id, duration)
-                or self._extract_chapters_from_description(description, duration))
+    def _extract_yt_initial_variable(self, webpage, regex, video_id, name):
+        return self._parse_json(self._search_regex(
+            (r'%s\s*%s' % (regex, self._YT_INITIAL_BOUNDARY_RE),
+             regex), webpage, name, default='{}'), video_id, fatal=False)
 
     def _real_extract(self, url):
         url, smuggled_data = unsmuggle_url(url, {})
+        video_id = self._match_id(url)
+        base_url = self.http_scheme() + '//www.youtube.com/'
+        webpage_url = base_url + 'watch?v=' + video_id
+        webpage = self._download_webpage(webpage_url, video_id, fatal=False)
 
-        proto = (
-            'http' if self._downloader.params.get('prefer_insecure', False)
-            else 'https')
+        player_response = None
+        if webpage:
+            player_response = self._extract_yt_initial_variable(
+                webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE,
+                video_id, 'initial player response')
+        if not player_response:
+            player_response = self._call_api(
+                'player', {'videoId': video_id}, video_id)
 
-        start_time = None
-        end_time = None
-        parsed_url = compat_urllib_parse_urlparse(url)
-        for component in [parsed_url.fragment, parsed_url.query]:
-            query = compat_parse_qs(component)
-            if start_time is None and 't' in query:
-                start_time = parse_duration(query['t'][0])
-            if start_time is None and 'start' in query:
-                start_time = parse_duration(query['start'][0])
-            if end_time is None and 'end' in query:
-                end_time = parse_duration(query['end'][0])
+        playability_status = player_response.get('playabilityStatus') or {}
+        if playability_status.get('reason') == 'Sign in to confirm your age':
+            pr = self._parse_json(try_get(compat_parse_qs(
+                self._download_webpage(
+                    base_url + 'get_video_info', video_id,
+                    'Refetching age-gated info webpage',
+                    'unable to download video info webpage', query={
+                        'video_id': video_id,
+                    }, fatal=False)),
+                lambda x: x['player_response'][0],
+                compat_str) or '{}', video_id)
+            if pr:
+                player_response = pr
 
-        # Extract original video URL from URL with redirection, like age verification, using next_url parameter
-        mobj = re.search(self._NEXT_URL_RE, url)
-        if mobj:
-            url = proto + '://www.youtube.com/' + compat_urllib_parse_unquote(mobj.group(1)).lstrip('/')
-        video_id = self.extract_id(url)
+        trailer_video_id = try_get(
+            playability_status,
+            lambda x: x['errorScreen']['playerLegacyDesktopYpcTrailerRenderer']['trailerVideoId'],
+            compat_str)
+        if trailer_video_id:
+            return self.url_result(
+                trailer_video_id, self.ie_key(), trailer_video_id)
 
-        # Get video webpage
-        url = proto + '://www.youtube.com/watch?v=%s&gl=US&hl=en&has_verified=1&bpctr=9999999999' % video_id
-        video_webpage, urlh = self._download_webpage_handle(url, video_id)
-
-        qs = compat_parse_qs(compat_urllib_parse_urlparse(urlh.geturl()).query)
-        video_id = qs.get('v', [None])[0] or video_id
-
-        # Attempt to extract SWF player URL
-        mobj = re.search(r'swfConfig.*?"(https?:\\/\\/.*?watch.*?-.*?\.swf)"', video_webpage)
-        if mobj is not None:
-            player_url = re.sub(r'\\(.)', r'\1', mobj.group(1))
-        else:
-            player_url = None
-
-        dash_mpds = []
-
-        def add_dash_mpd(video_info):
-            dash_mpd = video_info.get('dashmpd')
-            if dash_mpd and dash_mpd[0] not in dash_mpds:
-                dash_mpds.append(dash_mpd[0])
-
-        def add_dash_mpd_pr(pl_response):
-            dash_mpd = url_or_none(try_get(
-                pl_response, lambda x: x['streamingData']['dashManifestUrl'],
-                compat_str))
-            if dash_mpd and dash_mpd not in dash_mpds:
-                dash_mpds.append(dash_mpd)
-
-        is_live = None
-        view_count = None
-
-        def extract_view_count(v_info):
-            return int_or_none(try_get(v_info, lambda x: x['view_count'][0]))
-
-        def extract_player_response(player_response, video_id):
-            pl_response = str_or_none(player_response)
-            if not pl_response:
+        def get_text(x):
+            if not x:
                 return
-            pl_response = self._parse_json(pl_response, video_id, fatal=False)
-            if isinstance(pl_response, dict):
-                add_dash_mpd_pr(pl_response)
-                return pl_response
+            return x.get('simpleText') or ''.join([r['text'] for r in x['runs']])
 
-        player_response = {}
-
-        # Get video info
-        video_info = {}
-        embed_webpage = None
-        ytplayer_config = None
-
-        if re.search(r'["\']status["\']\s*:\s*["\']LOGIN_REQUIRED', video_webpage) is not None:
-            age_gate = True
-            # We simulate the access to the video from www.youtube.com/v/{video_id}
-            # this can be viewed without login into Youtube
-            url = proto + '://www.youtube.com/embed/%s' % video_id
-            embed_webpage = self._download_webpage(url, video_id, 'Downloading embed webpage')
-            data = compat_urllib_parse_urlencode({
-                'video_id': video_id,
-                'eurl': 'https://youtube.googleapis.com/v/' + video_id,
-                'sts': self._search_regex(
-                    r'"sts"\s*:\s*(\d+)', embed_webpage, 'sts', default=''),
-            })
-            video_info_url = proto + '://www.youtube.com/get_video_info?' + data
-            try:
-                video_info_webpage = self._download_webpage(
-                    video_info_url, video_id,
-                    note='Refetching age-gated info webpage',
-                    errnote='unable to download video info webpage')
-            except ExtractorError:
-                video_info_webpage = None
-            if video_info_webpage:
-                video_info = compat_parse_qs(video_info_webpage)
-                pl_response = video_info.get('player_response', [None])[0]
-                player_response = extract_player_response(pl_response, video_id)
-                add_dash_mpd(video_info)
-                view_count = extract_view_count(video_info)
-        else:
-            age_gate = False
-            # Try looking directly into the video webpage
-            ytplayer_config = self._get_ytplayer_config(video_id, video_webpage)
-            if ytplayer_config:
-                args = ytplayer_config['args']
-                if args.get('url_encoded_fmt_stream_map') or args.get('hlsvp'):
-                    # Convert to the same format returned by compat_parse_qs
-                    video_info = dict((k, [v]) for k, v in args.items())
-                    add_dash_mpd(video_info)
-                # Rental video is not rented but preview is available (e.g.
-                # https://www.youtube.com/watch?v=yYr8q0y5Jfg,
-                # https://github.com/ytdl-org/youtube-dl/issues/10532)
-                if not video_info and args.get('ypc_vid'):
-                    return self.url_result(
-                        args['ypc_vid'], YoutubeIE.ie_key(), video_id=args['ypc_vid'])
-                if args.get('livestream') == '1' or args.get('live_playback') == 1:
-                    is_live = True
-                if not player_response:
-                    player_response = extract_player_response(args.get('player_response'), video_id)
-            if not video_info or self._downloader.params.get('youtube_include_dash_manifest', True):
-                add_dash_mpd_pr(player_response)
-
-        if not video_info and not player_response:
-            player_response = extract_player_response(
-                self._search_regex(
-                    (r'%s\s*%s' % (self._YT_INITIAL_PLAYER_RESPONSE_RE, self._YT_INITIAL_BOUNDARY_RE),
-                     self._YT_INITIAL_PLAYER_RESPONSE_RE), video_webpage,
-                    'initial player response', default='{}'),
-                video_id)
-
-        def extract_unavailable_message():
-            messages = []
-            for tag, kind in (('h1', 'message'), ('div', 'submessage')):
-                msg = self._html_search_regex(
-                    r'(?s)<{tag}[^>]+id=["\']unavailable-{kind}["\'][^>]*>(.+?)</{tag}>'.format(tag=tag, kind=kind),
-                    video_webpage, 'unavailable %s' % kind, default=None)
-                if msg:
-                    messages.append(msg)
-            if messages:
-                return '\n'.join(messages)
-
-        if not video_info and not player_response:
-            unavailable_message = extract_unavailable_message()
-            if not unavailable_message:
-                unavailable_message = 'Unable to extract video data'
-            raise ExtractorError(
-                'YouTube said: %s' % unavailable_message, expected=True, video_id=video_id)
-
-        if not isinstance(video_info, dict):
-            video_info = {}
-
-        video_details = try_get(
-            player_response, lambda x: x['videoDetails'], dict) or {}
+        search_meta = (
+            lambda x: self._html_search_meta(x, webpage, default=None)) \
+            if webpage else lambda x: None
 
+        video_details = player_response.get('videoDetails') or {}
         microformat = try_get(
-            player_response, lambda x: x['microformat']['playerMicroformatRenderer'], dict) or {}
-
-        video_title = video_info.get('title', [None])[0] or video_details.get('title')
-        if not video_title:
-            self._downloader.report_warning('Unable to extract video title')
-            video_title = '_'
-
-        description_original = video_description = get_element_by_id("eow-description", video_webpage)
-        if video_description:
-
-            def replace_url(m):
-                redir_url = compat_urlparse.urljoin(url, m.group(1))
-                parsed_redir_url = compat_urllib_parse_urlparse(redir_url)
-                if re.search(r'^(?:www\.)?(?:youtube(?:-nocookie)?\.com|youtu\.be)$', parsed_redir_url.netloc) and parsed_redir_url.path == '/redirect':
-                    qs = compat_parse_qs(parsed_redir_url.query)
-                    q = qs.get('q')
-                    if q and q[0]:
-                        return q[0]
-                return redir_url
-
-            description_original = video_description = re.sub(r'''(?x)
-                <a\s+
-                    (?:[a-zA-Z-]+="[^"]*"\s+)*?
-                    (?:title|href)="([^"]+)"\s+
-                    (?:[a-zA-Z-]+="[^"]*"\s+)*?
-                    class="[^"]*"[^>]*>
-                [^<]+\.{3}\s*
-                </a>
-            ''', replace_url, video_description)
-            video_description = clean_html(video_description)
-        else:
-            video_description = video_details.get('shortDescription')
-            if video_description is None:
-                video_description = self._html_search_meta('description', video_webpage)
+            player_response,
+            lambda x: x['microformat']['playerMicroformatRenderer'],
+            dict) or {}
+        video_title = video_details.get('title') \
+            or get_text(microformat.get('title')) \
+            or search_meta(['og:title', 'twitter:title', 'title'])
+        video_description = video_details.get('shortDescription')
 
         if not smuggled_data.get('force_singlefeed', False):
             if not self._downloader.params.get('noplaylist'):
                 multifeed_metadata_list = try_get(
                     player_response,
                     lambda x: x['multicamera']['playerLegacyMulticameraRenderer']['metadataList'],
-                    compat_str) or try_get(
-                    video_info, lambda x: x['multifeed_metadata_list'][0], compat_str)
+                    compat_str)
                 if multifeed_metadata_list:
                     entries = []
                     feed_ids = []
@@ -1821,10 +1298,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         # Unquote should take place before split on comma (,) since textual
                         # fields may contain comma as well (see
                         # https://github.com/ytdl-org/youtube-dl/issues/8536)
-                        feed_data = compat_parse_qs(compat_urllib_parse_unquote_plus(feed))
+                        feed_data = compat_parse_qs(
+                            compat_urllib_parse_unquote_plus(feed))
 
                         def feed_entry(name):
-                            return try_get(feed_data, lambda x: x[name][0], compat_str)
+                            return try_get(
+                                feed_data, lambda x: x[name][0], compat_str)
 
                         feed_id = feed_entry('id')
                         if not feed_id:
@@ -1837,7 +1316,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             '_type': 'url_transparent',
                             'ie_key': 'Youtube',
                             'url': smuggle_url(
-                                '%s://www.youtube.com/watch?v=%s' % (proto, feed_data['id'][0]),
+                                base_url + 'watch?v=' + feed_data['id'][0],
                                 {'force_singlefeed': True}),
                             'title': title,
                         })
@@ -1845,631 +1324,393 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     self.to_screen(
                         'Downloading multifeed video (%s) - add --no-playlist to just download video %s'
                         % (', '.join(feed_ids), video_id))
-                    return self.playlist_result(entries, video_id, video_title, video_description)
+                    return self.playlist_result(
+                        entries, video_id, video_title, video_description)
             else:
                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
 
-        if view_count is None:
-            view_count = extract_view_count(video_info)
-        if view_count is None and video_details:
-            view_count = int_or_none(video_details.get('viewCount'))
-        if view_count is None and microformat:
-            view_count = int_or_none(microformat.get('viewCount'))
+        formats = []
+        itags = []
+        player_url = None
+        q = qualities(['tiny', 'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'])
+        streaming_data = player_response.get('streamingData') or {}
+        streaming_formats = streaming_data.get('formats') or []
+        streaming_formats.extend(streaming_data.get('adaptiveFormats') or [])
+        for fmt in streaming_formats:
+            if fmt.get('targetDurationSec') or fmt.get('drmFamilies'):
+                continue
 
-        if is_live is None:
-            is_live = bool_or_none(video_details.get('isLive'))
-
-        # Check for "rental" videos
-        if 'ypc_video_rental_bar_text' in video_info and 'author' not in video_info:
-            raise ExtractorError('"rental" videos not supported. See https://github.com/ytdl-org/youtube-dl/issues/359 for more information.', expected=True)
-
-        def _extract_filesize(media_url):
-            return int_or_none(self._search_regex(
-                r'\bclen[=/](\d+)', media_url, 'filesize', default=None))
-
-        streaming_formats = try_get(player_response, lambda x: x['streamingData']['formats'], list) or []
-        streaming_formats.extend(try_get(player_response, lambda x: x['streamingData']['adaptiveFormats'], list) or [])
-
-        if 'conn' in video_info and video_info['conn'][0].startswith('rtmp'):
-            self.report_rtmp_download()
-            formats = [{
-                'format_id': '_rtmp',
-                'protocol': 'rtmp',
-                'url': video_info['conn'][0],
-                'player_url': player_url,
-            }]
-        elif not is_live and (streaming_formats or len(video_info.get('url_encoded_fmt_stream_map', [''])[0]) >= 1 or len(video_info.get('adaptive_fmts', [''])[0]) >= 1):
-            encoded_url_map = video_info.get('url_encoded_fmt_stream_map', [''])[0] + ',' + video_info.get('adaptive_fmts', [''])[0]
-            if 'rtmpe%3Dyes' in encoded_url_map:
-                raise ExtractorError('rtmpe downloads are not supported, see https://github.com/ytdl-org/youtube-dl/issues/343 for more information.', expected=True)
-            formats = []
-            formats_spec = {}
-            fmt_list = video_info.get('fmt_list', [''])[0]
-            if fmt_list:
-                for fmt in fmt_list.split(','):
-                    spec = fmt.split('/')
-                    if len(spec) > 1:
-                        width_height = spec[1].split('x')
-                        if len(width_height) == 2:
-                            formats_spec[spec[0]] = {
-                                'resolution': spec[1],
-                                'width': int_or_none(width_height[0]),
-                                'height': int_or_none(width_height[1]),
-                            }
-            for fmt in streaming_formats:
-                itag = str_or_none(fmt.get('itag'))
-                if not itag:
+            fmt_url = fmt.get('url')
+            if not fmt_url:
+                sc = compat_parse_qs(fmt.get('signatureCipher'))
+                fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
+                encrypted_sig = try_get(sc, lambda x: x['s'][0])
+                if not (sc and fmt_url and encrypted_sig):
                     continue
-                quality = fmt.get('quality')
-                quality_label = fmt.get('qualityLabel') or quality
-                formats_spec[itag] = {
-                    'asr': int_or_none(fmt.get('audioSampleRate')),
-                    'filesize': int_or_none(fmt.get('contentLength')),
-                    'format_note': quality_label,
-                    'fps': int_or_none(fmt.get('fps')),
-                    'height': int_or_none(fmt.get('height')),
-                    # bitrate for itag 43 is always 2147483647
-                    'tbr': float_or_none(fmt.get('averageBitrate') or fmt.get('bitrate'), 1000) if itag != '43' else None,
-                    'width': int_or_none(fmt.get('width')),
-                }
-
-            for fmt in streaming_formats:
-                if fmt.get('drmFamilies') or fmt.get('drm_families'):
-                    continue
-                url = url_or_none(fmt.get('url'))
-
-                if not url:
-                    cipher = fmt.get('cipher') or fmt.get('signatureCipher')
-                    if not cipher:
+                if not player_url:
+                    if not webpage:
                         continue
-                    url_data = compat_parse_qs(cipher)
-                    url = url_or_none(try_get(url_data, lambda x: x['url'][0], compat_str))
-                    if not url:
+                    player_url = self._search_regex(
+                        r'"(?:PLAYER_JS_URL|jsUrl)"\s*:\s*"([^"]+)"',
+                        webpage, 'player URL', fatal=False)
+                if not player_url:
+                    continue
+                signature = self._decrypt_signature(sc['s'][0], video_id, player_url)
+                sp = try_get(sc, lambda x: x['sp'][0]) or 'signature'
+                fmt_url += '&' + sp + '=' + signature
+
+            itag = str_or_none(fmt.get('itag'))
+            if itag:
+                itags.append(itag)
+            quality = fmt.get('quality')
+            dct = {
+                'asr': int_or_none(fmt.get('audioSampleRate')),
+                'filesize': int_or_none(fmt.get('contentLength')),
+                'format_id': itag,
+                'format_note': fmt.get('qualityLabel') or quality,
+                'fps': int_or_none(fmt.get('fps')),
+                'height': int_or_none(fmt.get('height')),
+                'quality': q(quality),
+                'tbr': float_or_none(fmt.get(
+                    'averageBitrate') or fmt.get('bitrate'), 1000),
+                'url': fmt_url,
+                'width': fmt.get('width'),
+            }
+            mimetype = fmt.get('mimeType')
+            if mimetype:
+                mobj = re.match(
+                    r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', mimetype)
+                if mobj:
+                    dct['ext'] = mimetype2ext(mobj.group(1))
+                    dct.update(parse_codecs(mobj.group(2)))
+            if dct.get('acodec') == 'none' or dct.get('vcodec') == 'none':
+                dct['downloader_options'] = {
+                    # Youtube throttles chunks >~10M
+                    'http_chunk_size': 10485760,
+                }
+            formats.append(dct)
+
+        hls_manifest_url = streaming_data.get('hlsManifestUrl')
+        if hls_manifest_url:
+            for f in self._extract_m3u8_formats(
+                    hls_manifest_url, video_id, 'mp4', fatal=False):
+                itag = self._search_regex(
+                    r'/itag/(\d+)', f['url'], 'itag', default=None)
+                if itag:
+                    f['format_id'] = itag
+                formats.append(f)
+
+        if self._downloader.params.get('youtube_include_dash_manifest'):
+            dash_manifest_url = streaming_data.get('dashManifestUrl')
+            if dash_manifest_url:
+                for f in self._extract_mpd_formats(
+                        dash_manifest_url, video_id, fatal=False):
+                    if f['format_id'] in itags:
                         continue
-                else:
-                    cipher = None
-                    url_data = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+                    filesize = int_or_none(self._search_regex(
+                        r'/clen/(\d+)', f.get('fragment_base_url')
+                        or f['url'], 'file size', default=None))
+                    if filesize:
+                        f['filesize'] = filesize
+                    formats.append(f)
 
-                stream_type = int_or_none(try_get(url_data, lambda x: x['stream_type'][0]))
-                # Unsupported FORMAT_STREAM_TYPE_OTF
-                if stream_type == 3:
-                    continue
+        if not formats:
+            if streaming_data.get('licenseInfos'):
+                raise ExtractorError(
+                    'This video is DRM protected.', expected=True)
+            pemr = try_get(
+                playability_status,
+                lambda x: x['errorScreen']['playerErrorMessageRenderer'],
+                dict) or {}
+            reason = get_text(pemr.get('reason')) or playability_status.get('reason')
+            subreason = pemr.get('subreason')
+            if subreason:
+                subreason = clean_html(get_text(subreason))
+                if subreason == 'The uploader has not made this video available in your country.':
+                    countries = microformat.get('availableCountries')
+                    if not countries:
+                        regions_allowed = search_meta('regionsAllowed')
+                        countries = regions_allowed.split(',') if regions_allowed else None
+                    self.raise_geo_restricted(
+                        subreason, countries)
+                reason += '\n' + subreason
+            if reason:
+                raise ExtractorError(reason, expected=True)
 
-                format_id = fmt.get('itag') or url_data['itag'][0]
-                if not format_id:
-                    continue
-                format_id = compat_str(format_id)
+        self._sort_formats(formats)
 
-                if cipher:
-                    if 's' in url_data or self._downloader.params.get('youtube_include_dash_manifest', True):
-                        ASSETS_RE = (
-                            r'<script[^>]+\bsrc=("[^"]+")[^>]+\bname=["\']player_ias/base',
-                            r'"jsUrl"\s*:\s*("[^"]+")',
-                            r'"assets":.+?"js":\s*("[^"]+")')
-                        jsplayer_url_json = self._search_regex(
-                            ASSETS_RE,
-                            embed_webpage if age_gate else video_webpage,
-                            'JS player URL (1)', default=None)
-                        if not jsplayer_url_json and not age_gate:
-                            # We need the embed website after all
-                            if embed_webpage is None:
-                                embed_url = proto + '://www.youtube.com/embed/%s' % video_id
-                                embed_webpage = self._download_webpage(
-                                    embed_url, video_id, 'Downloading embed webpage')
-                            jsplayer_url_json = self._search_regex(
-                                ASSETS_RE, embed_webpage, 'JS player URL')
-
-                        player_url = json.loads(jsplayer_url_json)
-                        if player_url is None:
-                            player_url_json = self._search_regex(
-                                r'ytplayer\.config.*?"url"\s*:\s*("[^"]+")',
-                                video_webpage, 'age gate player URL')
-                            player_url = json.loads(player_url_json)
-
-                    if 'sig' in url_data:
-                        url += '&signature=' + url_data['sig'][0]
-                    elif 's' in url_data:
-                        encrypted_sig = url_data['s'][0]
-
-                        if self._downloader.params.get('verbose'):
-                            if player_url is None:
-                                player_desc = 'unknown'
-                            else:
-                                player_type, player_version = self._extract_player_info(player_url)
-                                player_desc = '%s player %s' % ('flash' if player_type == 'swf' else 'html5', player_version)
-                            parts_sizes = self._signature_cache_id(encrypted_sig)
-                            self.to_screen('{%s} signature length %s, %s' %
-                                           (format_id, parts_sizes, player_desc))
-
-                        signature = self._decrypt_signature(
-                            encrypted_sig, video_id, player_url, age_gate)
-                        sp = try_get(url_data, lambda x: x['sp'][0], compat_str) or 'signature'
-                        url += '&%s=%s' % (sp, signature)
-                if 'ratebypass' not in url:
-                    url += '&ratebypass=yes'
-
-                dct = {
-                    'format_id': format_id,
-                    'url': url,
-                    'player_url': player_url,
-                }
-                if format_id in self._formats:
-                    dct.update(self._formats[format_id])
-                if format_id in formats_spec:
-                    dct.update(formats_spec[format_id])
-
-                # Some itags are not included in DASH manifest thus corresponding formats will
-                # lack metadata (see https://github.com/ytdl-org/youtube-dl/pull/5993).
-                # Trying to extract metadata from url_encoded_fmt_stream_map entry.
-                mobj = re.search(r'^(?P<width>\d+)[xX](?P<height>\d+)$', url_data.get('size', [''])[0])
-                width, height = (int(mobj.group('width')), int(mobj.group('height'))) if mobj else (None, None)
-
-                if width is None:
-                    width = int_or_none(fmt.get('width'))
-                if height is None:
-                    height = int_or_none(fmt.get('height'))
-
-                filesize = int_or_none(url_data.get(
-                    'clen', [None])[0]) or _extract_filesize(url)
-
-                quality = url_data.get('quality', [None])[0] or fmt.get('quality')
-                quality_label = url_data.get('quality_label', [None])[0] or fmt.get('qualityLabel')
-
-                tbr = (float_or_none(url_data.get('bitrate', [None])[0], 1000)
-                       or float_or_none(fmt.get('bitrate'), 1000)) if format_id != '43' else None
-                fps = int_or_none(url_data.get('fps', [None])[0]) or int_or_none(fmt.get('fps'))
-
-                more_fields = {
-                    'filesize': filesize,
-                    'tbr': tbr,
-                    'width': width,
-                    'height': height,
-                    'fps': fps,
-                    'format_note': quality_label or quality,
-                }
-                for key, value in more_fields.items():
-                    if value:
-                        dct[key] = value
-                type_ = url_data.get('type', [None])[0] or fmt.get('mimeType')
-                if type_:
-                    type_split = type_.split(';')
-                    kind_ext = type_split[0].split('/')
-                    if len(kind_ext) == 2:
-                        kind, _ = kind_ext
-                        dct['ext'] = mimetype2ext(type_split[0])
-                        if kind in ('audio', 'video'):
-                            codecs = None
-                            for mobj in re.finditer(
-                                    r'(?P<key>[a-zA-Z_-]+)=(?P<quote>["\']?)(?P<val>.+?)(?P=quote)(?:;|$)', type_):
-                                if mobj.group('key') == 'codecs':
-                                    codecs = mobj.group('val')
-                                    break
-                            if codecs:
-                                dct.update(parse_codecs(codecs))
-                if dct.get('acodec') == 'none' or dct.get('vcodec') == 'none':
-                    dct['downloader_options'] = {
-                        # Youtube throttles chunks >~10M
-                        'http_chunk_size': 10485760,
-                    }
-                formats.append(dct)
-        else:
-            manifest_url = (
-                url_or_none(try_get(
-                    player_response,
-                    lambda x: x['streamingData']['hlsManifestUrl'],
-                    compat_str))
-                or url_or_none(try_get(
-                    video_info, lambda x: x['hlsvp'][0], compat_str)))
-            if manifest_url:
-                formats = []
-                m3u8_formats = self._extract_m3u8_formats(
-                    manifest_url, video_id, 'mp4', fatal=False)
-                for a_format in m3u8_formats:
-                    itag = self._search_regex(
-                        r'/itag/(\d+)/', a_format['url'], 'itag', default=None)
-                    if itag:
-                        a_format['format_id'] = itag
-                        if itag in self._formats:
-                            dct = self._formats[itag].copy()
-                            dct.update(a_format)
-                            a_format = dct
-                    a_format['player_url'] = player_url
-                    # Accept-Encoding header causes failures in live streams on Youtube and Youtube Gaming
-                    a_format.setdefault('http_headers', {})['Youtubedl-no-compression'] = 'True'
-                    formats.append(a_format)
-            else:
-                error_message = extract_unavailable_message()
-                if not error_message:
-                    reason_list = try_get(
-                        player_response,
-                        lambda x: x['playabilityStatus']['errorScreen']['playerErrorMessageRenderer']['subreason']['runs'],
-                        list) or []
-                    for reason in reason_list:
-                        if not isinstance(reason, dict):
-                            continue
-                        reason_text = try_get(reason, lambda x: x['text'], compat_str)
-                        if reason_text:
-                            if not error_message:
-                                error_message = ''
-                            error_message += reason_text
-                    if error_message:
-                        error_message = clean_html(error_message)
-                if not error_message:
-                    error_message = clean_html(try_get(
-                        player_response, lambda x: x['playabilityStatus']['reason'],
-                        compat_str))
-                if not error_message:
-                    error_message = clean_html(
-                        try_get(video_info, lambda x: x['reason'][0], compat_str))
-                if error_message:
-                    raise ExtractorError(error_message, expected=True)
-                raise ExtractorError('no conn, hlsvp, hlsManifestUrl or url_encoded_fmt_stream_map information found in video info')
-
-        # uploader
-        video_uploader = try_get(
-            video_info, lambda x: x['author'][0],
-            compat_str) or str_or_none(video_details.get('author'))
-        if video_uploader:
-            video_uploader = compat_urllib_parse_unquote_plus(video_uploader)
-        else:
-            self._downloader.report_warning('unable to extract uploader name')
-
-        # uploader_id
-        video_uploader_id = None
-        video_uploader_url = None
-        mobj = re.search(
-            r'<link itemprop="url" href="(?P<uploader_url>https?://www\.youtube\.com/(?:user|channel)/(?P<uploader_id>[^"]+))">',
-            video_webpage)
-        if mobj is not None:
-            video_uploader_id = mobj.group('uploader_id')
-            video_uploader_url = mobj.group('uploader_url')
-        else:
-            owner_profile_url = url_or_none(microformat.get('ownerProfileUrl'))
-            if owner_profile_url:
-                video_uploader_id = self._search_regex(
-                    r'(?:user|channel)/([^/]+)', owner_profile_url, 'uploader id',
-                    default=None)
-                video_uploader_url = owner_profile_url
-
-        channel_id = (
-            str_or_none(video_details.get('channelId'))
-            or self._html_search_meta(
-                'channelId', video_webpage, 'channel id', default=None)
-            or self._search_regex(
-                r'data-channel-external-id=(["\'])(?P<id>(?:(?!\1).)+)\1',
-                video_webpage, 'channel id', default=None, group='id'))
-        channel_url = 'http://www.youtube.com/channel/%s' % channel_id if channel_id else None
+        keywords = video_details.get('keywords') or []
+        if not keywords and webpage:
+            keywords = [
+                unescapeHTML(m.group('content'))
+                for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
+        for keyword in keywords:
+            if keyword.startswith('yt:stretch='):
+                w, h = keyword.split('=')[1].split(':')
+                w, h = int(w), int(h)
+                if w > 0 and h > 0:
+                    ratio = w / h
+                    for f in formats:
+                        if f.get('vcodec') != 'none':
+                            f['stretched_ratio'] = ratio
 
         thumbnails = []
-        thumbnails_list = try_get(
-            video_details, lambda x: x['thumbnail']['thumbnails'], list) or []
-        for t in thumbnails_list:
-            if not isinstance(t, dict):
-                continue
-            thumbnail_url = url_or_none(t.get('url'))
-            if not thumbnail_url:
-                continue
-            thumbnails.append({
-                'url': thumbnail_url,
-                'width': int_or_none(t.get('width')),
-                'height': int_or_none(t.get('height')),
-            })
-
-        if not thumbnails:
-            video_thumbnail = None
-            # We try first to get a high quality image:
-            m_thumb = re.search(r'<span itemprop="thumbnail".*?href="(.*?)">',
-                                video_webpage, re.DOTALL)
-            if m_thumb is not None:
-                video_thumbnail = m_thumb.group(1)
-            thumbnail_url = try_get(video_info, lambda x: x['thumbnail_url'][0], compat_str)
-            if thumbnail_url:
-                video_thumbnail = compat_urllib_parse_unquote_plus(thumbnail_url)
-            if video_thumbnail:
-                thumbnails.append({'url': video_thumbnail})
-
-        # upload date
-        upload_date = self._html_search_meta(
-            'datePublished', video_webpage, 'upload date', default=None)
-        if not upload_date:
-            upload_date = self._search_regex(
-                [r'(?s)id="eow-date.*?>(.*?)</span>',
-                 r'(?:id="watch-uploader-info".*?>.*?|["\']simpleText["\']\s*:\s*["\'])(?:Published|Uploaded|Streamed live|Started) on (.+?)[<"\']'],
-                video_webpage, 'upload date', default=None)
-        if not upload_date:
-            upload_date = microformat.get('publishDate') or microformat.get('uploadDate')
-        upload_date = unified_strdate(upload_date)
-
-        video_license = self._html_search_regex(
-            r'<h4[^>]+class="title"[^>]*>\s*License\s*</h4>\s*<ul[^>]*>\s*<li>(.+?)</li',
-            video_webpage, 'license', default=None)
-
-        m_music = re.search(
-            r'''(?x)
-                <h4[^>]+class="title"[^>]*>\s*Music\s*</h4>\s*
-                <ul[^>]*>\s*
-                <li>(?P<title>.+?)
-                by (?P<creator>.+?)
-                (?:
-                    \(.+?\)|
-                    <a[^>]*
-                        (?:
-                            \bhref=["\']/red[^>]*>|             # drop possible
-                            >\s*Listen ad-free with YouTube Red # YouTube Red ad
-                        )
-                    .*?
-                )?</li
-            ''',
-            video_webpage)
-        if m_music:
-            video_alt_title = remove_quotes(unescapeHTML(m_music.group('title')))
-            video_creator = clean_html(m_music.group('creator'))
+        for container in (video_details, microformat):
+            for thumbnail in (try_get(
+                    container,
+                    lambda x: x['thumbnail']['thumbnails'], list) or []):
+                thumbnail_url = thumbnail.get('url')
+                if not thumbnail_url:
+                    continue
+                thumbnails.append({
+                    'height': int_or_none(thumbnail.get('height')),
+                    'url': thumbnail_url,
+                    'width': int_or_none(thumbnail.get('width')),
+                })
+            if thumbnails:
+                break
         else:
-            video_alt_title = video_creator = None
+            thumbnail = search_meta(['og:image', 'twitter:image'])
+            if thumbnail:
+                thumbnails = [{'url': thumbnail}]
 
-        def extract_meta(field):
-            return self._html_search_regex(
-                r'<h4[^>]+class="title"[^>]*>\s*%s\s*</h4>\s*<ul[^>]*>\s*<li>(.+?)</li>\s*' % field,
-                video_webpage, field, default=None)
+        category = microformat.get('category') or search_meta('genre')
+        channel_id = video_details.get('channelId') \
+            or microformat.get('externalChannelId') \
+            or search_meta('channelId')
+        duration = int_or_none(
+            video_details.get('lengthSeconds')
+            or microformat.get('lengthSeconds')) \
+            or parse_duration(search_meta('duration'))
+        is_live = video_details.get('isLive')
+        owner_profile_url = microformat.get('ownerProfileUrl')
 
-        track = extract_meta('Song')
-        artist = extract_meta('Artist')
-        album = extract_meta('Album')
+        info = {
+            'id': video_id,
+            'title': self._live_title(video_title) if is_live else video_title,
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'description': video_description,
+            'upload_date': unified_strdate(
+                microformat.get('uploadDate')
+                or search_meta('uploadDate')),
+            'uploader': video_details['author'],
+            'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
+            'uploader_url': owner_profile_url,
+            'channel_id': channel_id,
+            'channel_url': 'https://www.youtube.com/channel/' + channel_id if channel_id else None,
+            'duration': duration,
+            'view_count': int_or_none(
+                video_details.get('viewCount')
+                or microformat.get('viewCount')
+                or search_meta('interactionCount')),
+            'average_rating': float_or_none(video_details.get('averageRating')),
+            'age_limit': 18 if (
+                microformat.get('isFamilySafe') is False
+                or search_meta('isFamilyFriendly') == 'false'
+                or search_meta('og:restrictions:age') == '18+') else 0,
+            'webpage_url': webpage_url,
+            'categories': [category] if category else None,
+            'tags': keywords,
+            'is_live': is_live,
+        }
+
+        pctr = try_get(
+            player_response,
+            lambda x: x['captions']['playerCaptionsTracklistRenderer'], dict)
+        if pctr:
+            def process_language(container, base_url, caption, query):
+                lang_subs = []
+                for fmt in self._SUBTITLE_FORMATS:
+                    query.update({
+                        'fmt': fmt,
+                    })
+                    lang_subs.append({
+                        'ext': fmt,
+                        'url': update_url_query(base_url, query),
+                    })
+                subtitles[caption['languageCode']] = lang_subs
+
+            subtitles = {}
+            for caption_track in pctr['captionTracks']:
+                base_url = caption_track['baseUrl']
+                if caption_track.get('kind') != 'asr':
+                    lang_subs = []
+                    for fmt in self._SUBTITLE_FORMATS:
+                        lang_subs.append({
+                            'ext': fmt,
+                            'url': update_url_query(base_url, {
+                                'fmt': fmt,
+                            }),
+                        })
+                    subtitles[caption_track['languageCode']] = lang_subs
+                    continue
+                automatic_captions = {}
+                for translation_language in pctr['translationLanguages']:
+                    translation_language_code = translation_language['languageCode']
+                    lang_subs = []
+                    for fmt in self._SUBTITLE_FORMATS:
+                        lang_subs.append({
+                            'ext': fmt,
+                            'url': update_url_query(base_url, {
+                                'fmt': fmt,
+                                'tlang': translation_language_code,
+                            }),
+                        })
+                    automatic_captions[translation_language_code] = lang_subs
+                info['automatic_captions'] = automatic_captions
+            info['subtitles'] = subtitles
+
+        parsed_url = compat_urllib_parse_urlparse(url)
+        for component in [parsed_url.fragment, parsed_url.query]:
+            query = compat_parse_qs(component)
+            for k, v in query.items():
+                for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
+                    d_k += '_time'
+                    if d_k not in info and k in s_ks:
+                        info[d_k] = parse_duration(query[k][0])
 
-        # Youtube Music Auto-generated description
-        release_date = release_year = None
         if video_description:
             mobj = re.search(r'(?s)(?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?.+\nAuto-generated by YouTube\.\s*$', video_description)
             if mobj:
-                if not track:
-                    track = mobj.group('track').strip()
-                if not artist:
-                    artist = mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·'))
-                if not album:
-                    album = mobj.group('album'.strip())
                 release_year = mobj.group('release_year')
                 release_date = mobj.group('release_date')
                 if release_date:
                     release_date = release_date.replace('-', '')
                     if not release_year:
-                        release_year = int(release_date[:4])
-                if release_year:
-                    release_year = int(release_year)
+                        release_year = release_date[:4]
+                info.update({
+                    'album': mobj.group('album'.strip()),
+                    'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
+                    'track': mobj.group('track').strip(),
+                    'release_date': release_date,
+                    'release_year': int(release_year),
+                })
 
-        yt_initial_data = self._extract_yt_initial_data(video_id, video_webpage)
-        contents = try_get(yt_initial_data, lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'], list) or []
-        for content in contents:
-            rows = try_get(content, lambda x: x['videoSecondaryInfoRenderer']['metadataRowContainer']['metadataRowContainerRenderer']['rows'], list) or []
-            multiple_songs = False
-            for row in rows:
-                if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
-                    multiple_songs = True
-                    break
-            for row in rows:
-                mrr = row.get('metadataRowRenderer') or {}
-                mrr_title = try_get(
-                    mrr, lambda x: x['title']['simpleText'], compat_str)
-                mrr_contents = try_get(
-                    mrr, lambda x: x['contents'][0], dict) or {}
-                mrr_contents_text = try_get(mrr_contents, [lambda x: x['simpleText'], lambda x: x['runs'][0]['text']], compat_str)
-                if not (mrr_title and mrr_contents_text):
+        initial_data = None
+        if webpage:
+            initial_data = self._extract_yt_initial_variable(
+                webpage, self._YT_INITIAL_DATA_RE, video_id,
+                'yt initial data')
+        if not initial_data:
+            initial_data = self._call_api(
+                'next', {'videoId': video_id}, video_id, fatal=False)
+
+        if initial_data:
+            for engagment_pannel in (initial_data.get('engagementPanels') or []):
+                contents = try_get(
+                    engagment_pannel, lambda x: x['engagementPanelSectionListRenderer']['content']['macroMarkersListRenderer']['contents'],
+                    list)
+                if not contents:
                     continue
-                if mrr_title == 'License':
-                    video_license = mrr_contents_text
-                elif not multiple_songs:
-                    if mrr_title == 'Album':
-                        album = mrr_contents_text
-                    elif mrr_title == 'Artist':
-                        artist = mrr_contents_text
-                    elif mrr_title == 'Song':
-                        track = mrr_contents_text
 
-        m_episode = re.search(
-            r'<div[^>]+id="watch7-headline"[^>]*>\s*<span[^>]*>.*?>(?P<series>[^<]+)</a></b>\s*S(?P<season>\d+)\s*•\s*E(?P<episode>\d+)</span>',
-            video_webpage)
-        if m_episode:
-            series = unescapeHTML(m_episode.group('series'))
-            season_number = int(m_episode.group('season'))
-            episode_number = int(m_episode.group('episode'))
-        else:
-            series = season_number = episode_number = None
+                def chapter_time(mmlir):
+                    return parse_duration(mmlir.get(
+                        get_text(mmlir.get('timeDescription'))))
 
-        m_cat_container = self._search_regex(
-            r'(?s)<h4[^>]*>\s*Category\s*</h4>\s*<ul[^>]*>(.*?)</ul>',
-            video_webpage, 'categories', default=None)
-        category = None
-        if m_cat_container:
-            category = self._html_search_regex(
-                r'(?s)<a[^<]+>(.*?)</a>', m_cat_container, 'category',
-                default=None)
-        if not category:
-            category = try_get(
-                microformat, lambda x: x['category'], compat_str)
-        video_categories = None if category is None else [category]
+                chapters = []
+                for next_num, content in enumerate(contents, start=1):
+                    mmlir = content.get('macroMarkersListItemRenderer') or {}
+                    start_time = chapter_time(mmlir)
+                    end_time = chapter_time(try_get(
+                        contents, lambda x: x[next_num]['macroMarkersListItemRenderer'])) \
+                        if next_num < len(contents) else duration
+                    if not (start_time and end_time):
+                        continue
+                    chapters.append({
+                        'start_time': start_time,
+                        'end_time': end_time,
+                        'title': get_text(mmlir.get('title')),
+                    })
+                info['chapters'] = chapters
 
-        video_tags = [
-            unescapeHTML(m.group('content'))
-            for m in re.finditer(self._meta_regex('og:video:tag'), video_webpage)]
-        if not video_tags:
-            video_tags = try_get(video_details, lambda x: x['keywords'], list)
+            contents = try_get(
+                initial_data,
+                lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'],
+                list) or []
+            for content in contents:
+                vpir = content.get('videoPrimaryInfoRenderer')
+                if vpir:
+                    stl = vpir.get('superTitleLink')
+                    if stl:
+                        stl = get_text(stl)
+                        if try_get(
+                                vpir,
+                                lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
+                            info['location'] = stl
+                        else:
+                            mobj = re.search(r'(.+?)\s*S(\d+)\s*•\s*E(\d+)', stl)
+                            if mobj:
+                                info.update({
+                                    'series': mobj.group(1),
+                                    'season_number': int(mobj.group(2)),
+                                    'episode_number': int(mobj.group(3)),
+                                })
+                    for tlb in (try_get(
+                            vpir,
+                            lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
+                            list) or []):
+                        tbr = tlb.get('toggleButtonRenderer') or {}
+                        for getter, regex in [(
+                                lambda x: x['defaultText']['accessibility']['accessibilityData'],
+                                r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
+                                    lambda x: x['accessibility'],
+                                    lambda x: x['accessibilityData']['accessibilityData'],
+                                ], r'(?P<type>(?:dis)?like) this video along with (?P<count>[\d,]+) other people')]:
+                            label = (try_get(tbr, getter, dict) or {}).get('label')
+                            if label:
+                                mobj = re.match(regex, label)
+                                if mobj:
+                                    info[mobj.group('type') + '_count'] = str_to_int(mobj.group('count'))
+                                    break
+                    sbr_tooltip = try_get(
+                        vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
+                    if sbr_tooltip:
+                        like_count, dislike_count = sbr_tooltip.split(' / ')
+                        info.update({
+                            'like_count': str_to_int(like_count),
+                            'dislike_count': str_to_int(dislike_count),
+                        })
+                vsir = content.get('videoSecondaryInfoRenderer')
+                if vsir:
+                    info['channel'] = get_text(try_get(
+                        vsir,
+                        lambda x: x['owner']['videoOwnerRenderer']['title'],
+                        compat_str))
+                    rows = try_get(
+                        vsir,
+                        lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
+                        list) or []
+                    multiple_songs = False
+                    for row in rows:
+                        if try_get(row, lambda x: x['metadataRowRenderer']['hasDividerLine']) is True:
+                            multiple_songs = True
+                            break
+                    for row in rows:
+                        mrr = row.get('metadataRowRenderer') or {}
+                        mrr_title = mrr.get('title')
+                        if not mrr_title:
+                            continue
+                        mrr_title = get_text(mrr['title'])
+                        mrr_contents_text = get_text(mrr['contents'][0])
+                        if mrr_title == 'License':
+                            info['license'] = mrr_contents_text
+                        elif not multiple_songs:
+                            if mrr_title == 'Album':
+                                info['album'] = mrr_contents_text
+                            elif mrr_title == 'Artist':
+                                info['artist'] = mrr_contents_text
+                            elif mrr_title == 'Song':
+                                info['track'] = mrr_contents_text
 
-        def _extract_count(count_name):
-            return str_to_int(self._search_regex(
-                (r'-%s-button[^>]+><span[^>]+class="yt-uix-button-content"[^>]*>([\d,]+)</span>' % re.escape(count_name),
-                 r'["\']label["\']\s*:\s*["\']([\d,.]+)\s+%ss["\']' % re.escape(count_name)),
-                video_webpage, count_name, default=None))
+        for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
+            v = info.get(s_k)
+            if v:
+                info[d_k] = v
 
-        like_count = _extract_count('like')
-        dislike_count = _extract_count('dislike')
+        self.mark_watched(video_id, player_response)
 
-        if view_count is None:
-            view_count = str_to_int(self._search_regex(
-                r'<[^>]+class=["\']watch-view-count[^>]+>\s*([\d,\s]+)', video_webpage,
-                'view count', default=None))
-
-        average_rating = (
-            float_or_none(video_details.get('averageRating'))
-            or try_get(video_info, lambda x: float_or_none(x['avg_rating'][0])))
-
-        # subtitles
-        video_subtitles = self.extract_subtitles(video_id, video_webpage)
-        automatic_captions = self.extract_automatic_captions(video_id, player_response, ytplayer_config)
-
-        video_duration = try_get(
-            video_info, lambda x: int_or_none(x['length_seconds'][0]))
-        if not video_duration:
-            video_duration = int_or_none(video_details.get('lengthSeconds'))
-        if not video_duration:
-            video_duration = parse_duration(self._html_search_meta(
-                'duration', video_webpage, 'video duration'))
-
-        # annotations
-        video_annotations = None
-        if self._downloader.params.get('writeannotations', False):
-            xsrf_token = None
-            ytcfg = self._extract_ytcfg(video_id, video_webpage)
-            if ytcfg:
-                xsrf_token = try_get(ytcfg, lambda x: x['XSRF_TOKEN'], compat_str)
-            if not xsrf_token:
-                xsrf_token = self._search_regex(
-                    r'([\'"])XSRF_TOKEN\1\s*:\s*([\'"])(?P<xsrf_token>(?:(?!\2).)+)\2',
-                    video_webpage, 'xsrf token', group='xsrf_token', fatal=False)
-            invideo_url = try_get(
-                player_response, lambda x: x['annotations'][0]['playerAnnotationsUrlsRenderer']['invideoUrl'], compat_str)
-            if xsrf_token and invideo_url:
-                xsrf_field_name = None
-                if ytcfg:
-                    xsrf_field_name = try_get(ytcfg, lambda x: x['XSRF_FIELD_NAME'], compat_str)
-                if not xsrf_field_name:
-                    xsrf_field_name = self._search_regex(
-                        r'([\'"])XSRF_FIELD_NAME\1\s*:\s*([\'"])(?P<xsrf_field_name>\w+)\2',
-                        video_webpage, 'xsrf field name',
-                        group='xsrf_field_name', default='session_token')
-                video_annotations = self._download_webpage(
-                    self._proto_relative_url(invideo_url),
-                    video_id, note='Downloading annotations',
-                    errnote='Unable to download video annotations', fatal=False,
-                    data=urlencode_postdata({xsrf_field_name: xsrf_token}))
-
-        chapters = self._extract_chapters(video_webpage, description_original, video_id, video_duration)
-
-        # Look for the DASH manifest
-        if self._downloader.params.get('youtube_include_dash_manifest', True):
-            dash_mpd_fatal = True
-            for mpd_url in dash_mpds:
-                dash_formats = {}
-                try:
-                    def decrypt_sig(mobj):
-                        s = mobj.group(1)
-                        dec_s = self._decrypt_signature(s, video_id, player_url, age_gate)
-                        return '/signature/%s' % dec_s
-
-                    mpd_url = re.sub(r'/s/([a-fA-F0-9\.]+)', decrypt_sig, mpd_url)
-
-                    for df in self._extract_mpd_formats(
-                            mpd_url, video_id, fatal=dash_mpd_fatal,
-                            formats_dict=self._formats):
-                        if not df.get('filesize'):
-                            df['filesize'] = _extract_filesize(df.get('fragment_base_url') or df['url'])
-                        # Do not overwrite DASH format found in some previous DASH manifest
-                        if df['format_id'] not in dash_formats:
-                            dash_formats[df['format_id']] = df
-                        # Additional DASH manifests may end up in HTTP Error 403 therefore
-                        # allow them to fail without bug report message if we already have
-                        # some DASH manifest succeeded. This is temporary workaround to reduce
-                        # burst of bug reports until we figure out the reason and whether it
-                        # can be fixed at all.
-                        dash_mpd_fatal = False
-                except (ExtractorError, KeyError) as e:
-                    self.report_warning(
-                        'Skipping DASH manifest: %r' % e, video_id)
-                if dash_formats:
-                    # Remove the formats we found through non-DASH, they
-                    # contain less info and it can be wrong, because we use
-                    # fixed values (for example the resolution). See
-                    # https://github.com/ytdl-org/youtube-dl/issues/5774 for an
-                    # example.
-                    formats = [f for f in formats if f['format_id'] not in dash_formats.keys()]
-                    formats.extend(dash_formats.values())
-
-        # Check for malformed aspect ratio
-        stretched_m = re.search(
-            r'<meta\s+property="og:video:tag".*?content="yt:stretch=(?P<w>[0-9]+):(?P<h>[0-9]+)">',
-            video_webpage)
-        if stretched_m:
-            w = float(stretched_m.group('w'))
-            h = float(stretched_m.group('h'))
-            # yt:stretch may hold invalid ratio data (e.g. for Q39EVAstoRM ratio is 17:0).
-            # We will only process correct ratios.
-            if w > 0 and h > 0:
-                ratio = w / h
-                for f in formats:
-                    if f.get('vcodec') != 'none':
-                        f['stretched_ratio'] = ratio
-
-        if not formats:
-            if 'reason' in video_info:
-                if 'The uploader has not made this video available in your country.' in video_info['reason']:
-                    regions_allowed = self._html_search_meta(
-                        'regionsAllowed', video_webpage, default=None)
-                    countries = regions_allowed.split(',') if regions_allowed else None
-                    self.raise_geo_restricted(
-                        msg=video_info['reason'][0], countries=countries)
-                reason = video_info['reason'][0]
-                if 'Invalid parameters' in reason:
-                    unavailable_message = extract_unavailable_message()
-                    if unavailable_message:
-                        reason = unavailable_message
-                raise ExtractorError(
-                    'YouTube said: %s' % reason,
-                    expected=True, video_id=video_id)
-            if video_info.get('license_info') or try_get(player_response, lambda x: x['streamingData']['licenseInfos']):
-                raise ExtractorError('This video is DRM protected.', expected=True)
-
-        self._sort_formats(formats)
-
-        self.mark_watched(video_id, video_info, player_response)
-
-        return {
-            'id': video_id,
-            'uploader': video_uploader,
-            'uploader_id': video_uploader_id,
-            'uploader_url': video_uploader_url,
-            'channel_id': channel_id,
-            'channel_url': channel_url,
-            'upload_date': upload_date,
-            'license': video_license,
-            'creator': video_creator or artist,
-            'title': video_title,
-            'alt_title': video_alt_title or track,
-            'thumbnails': thumbnails,
-            'description': video_description,
-            'categories': video_categories,
-            'tags': video_tags,
-            'subtitles': video_subtitles,
-            'automatic_captions': automatic_captions,
-            'duration': video_duration,
-            'age_limit': 18 if age_gate else 0,
-            'annotations': video_annotations,
-            'chapters': chapters,
-            'webpage_url': proto + '://www.youtube.com/watch?v=%s' % video_id,
-            'view_count': view_count,
-            'like_count': like_count,
-            'dislike_count': dislike_count,
-            'average_rating': average_rating,
-            'formats': formats,
-            'is_live': is_live,
-            'start_time': start_time,
-            'end_time': end_time,
-            'series': series,
-            'season_number': season_number,
-            'episode_number': episode_number,
-            'track': track,
-            'artist': artist,
-            'album': album,
-            'release_date': release_date,
-            'release_year': release_year,
-        }
+        return info
 
 
 class YoutubeTabIE(YoutubeBaseInfoExtractor):

From 9c724601ba234085dc5071ec9c1c3d98e6834817 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 1 Feb 2021 16:08:50 +0100
Subject: [PATCH 170/860] [youtube] remove description chapters tests

video description no longer contain yt.www.watch.player.seekTo
function
---
 test/test_youtube_chapters.py   | 275 --------------------------------
 youtube_dl/extractor/youtube.py |  91 ++++++++---
 2 files changed, 67 insertions(+), 299 deletions(-)
 delete mode 100644 test/test_youtube_chapters.py

diff --git a/test/test_youtube_chapters.py b/test/test_youtube_chapters.py
deleted file mode 100644
index e69c57377..000000000
--- a/test/test_youtube_chapters.py
+++ /dev/null
@@ -1,275 +0,0 @@
-#!/usr/bin/env python
-# coding: utf-8
-from __future__ import unicode_literals
-
-# Allow direct execution
-import os
-import sys
-import unittest
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
-
-from test.helper import expect_value
-from youtube_dl.extractor import YoutubeIE
-
-
-class TestYoutubeChapters(unittest.TestCase):
-
-    _TEST_CASES = [
-        (
-            # https://www.youtube.com/watch?v=A22oy8dFjqc
-            # pattern: 00:00 - <title>
-            '''This is the absolute ULTIMATE experience of Queen's set at LIVE AID, this is the best video mixed to the absolutely superior stereo radio broadcast. This vastly superior audio mix takes a huge dump on all of the official mixes. Best viewed in 1080p. ENJOY! ***MAKE SURE TO READ THE DESCRIPTION***<br /><a href="#" onclick="yt.www.watch.player.seekTo(00*60+36);return false;">00:36</a> - Bohemian Rhapsody<br /><a href="#" onclick="yt.www.watch.player.seekTo(02*60+42);return false;">02:42</a> - Radio Ga Ga<br /><a href="#" onclick="yt.www.watch.player.seekTo(06*60+53);return false;">06:53</a> - Ay Oh!<br /><a href="#" onclick="yt.www.watch.player.seekTo(07*60+34);return false;">07:34</a> - Hammer To Fall<br /><a href="#" onclick="yt.www.watch.player.seekTo(12*60+08);return false;">12:08</a> - Crazy Little Thing Called Love<br /><a href="#" onclick="yt.www.watch.player.seekTo(16*60+03);return false;">16:03</a> - We Will Rock You<br /><a href="#" onclick="yt.www.watch.player.seekTo(17*60+18);return false;">17:18</a> - We Are The Champions<br /><a href="#" onclick="yt.www.watch.player.seekTo(21*60+12);return false;">21:12</a> - Is This The World We Created...?<br /><br />Short song analysis:<br /><br />- "Bohemian Rhapsody": Although it's a short medley version, it's one of the best performances of the ballad section, with Freddie nailing the Bb4s with the correct studio phrasing (for the first time ever!).<br /><br />- "Radio Ga Ga": Although it's missing one chorus, this is one of - if not the best - the best versions ever, Freddie nails all the Bb4s and sounds very clean! Spike Edney's Roland Jupiter 8 also really shines through on this mix, compared to the DVD releases!<br /><br />- "Audience Improv": A great improv, Freddie sounds strong and confident. You gotta love when he sustains that A4 for 4 seconds!<br /><br />- "Hammer To Fall": Despite missing a verse and a chorus, it's a strong version (possibly the best ever). Freddie sings the song amazingly, and even ad-libs a C#5 and a C5! Also notice how heavy Brian's guitar sounds compared to the thin DVD mixes - it roars!<br /><br />- "Crazy Little Thing Called Love": A great version, the crowd loves the song, the jam is great as well! Only downside to this is the slight feedback issues.<br /><br />- "We Will Rock You": Although cut down to the 1st verse and chorus, Freddie sounds strong. He nails the A4, and the solo from Dr. May is brilliant!<br /><br />- "We Are the Champions": Perhaps the high-light of the performance - Freddie is very daring on this version, he sustains the pre-chorus Bb4s, nails the 1st C5, belts great A4s, but most importantly: He nails the chorus Bb4s, in all 3 choruses! This is the only time he has ever done so! It has to be said though, the last one sounds a bit rough, but that's a side effect of belting high notes for the past 18 minutes, with nodules AND laryngitis!<br /><br />- "Is This The World We Created... ?": Freddie and Brian perform a beautiful version of this, and it is one of the best versions ever. It's both sad and hilarious that a couple of BBC engineers are talking over the song, one of them being completely oblivious of the fact that he is interrupting the performance, on live television... Which was being televised to almost 2 billion homes.<br /><br /><br />All rights go to their respective owners!<br />-----Copyright Disclaimer Under Section 107 of the Copyright Act 1976, allowance is made for fair use for purposes such as criticism, comment, news reporting, teaching, scholarship, and research. Fair use is a use permitted by copyright statute that might otherwise be infringing. Non-profit, educational or personal use tips the balance in favor of fair use''',
-            1477,
-            [{
-                'start_time': 36,
-                'end_time': 162,
-                'title': 'Bohemian Rhapsody',
-            }, {
-                'start_time': 162,
-                'end_time': 413,
-                'title': 'Radio Ga Ga',
-            }, {
-                'start_time': 413,
-                'end_time': 454,
-                'title': 'Ay Oh!',
-            }, {
-                'start_time': 454,
-                'end_time': 728,
-                'title': 'Hammer To Fall',
-            }, {
-                'start_time': 728,
-                'end_time': 963,
-                'title': 'Crazy Little Thing Called Love',
-            }, {
-                'start_time': 963,
-                'end_time': 1038,
-                'title': 'We Will Rock You',
-            }, {
-                'start_time': 1038,
-                'end_time': 1272,
-                'title': 'We Are The Champions',
-            }, {
-                'start_time': 1272,
-                'end_time': 1477,
-                'title': 'Is This The World We Created...?',
-            }]
-        ),
-        (
-            # https://www.youtube.com/watch?v=ekYlRhALiRQ
-            # pattern: <num>. <title> 0:00
-            '1.  Those Beaten Paths of Confusion <a href="#" onclick="yt.www.watch.player.seekTo(0*60+00);return false;">0:00</a><br />2.  Beyond the Shadows of Emptiness & Nothingness <a href="#" onclick="yt.www.watch.player.seekTo(11*60+47);return false;">11:47</a><br />3.  Poison Yourself...With Thought <a href="#" onclick="yt.www.watch.player.seekTo(26*60+30);return false;">26:30</a><br />4.  The Agents of Transformation <a href="#" onclick="yt.www.watch.player.seekTo(35*60+57);return false;">35:57</a><br />5.  Drowning in the Pain of Consciousness <a href="#" onclick="yt.www.watch.player.seekTo(44*60+32);return false;">44:32</a><br />6.  Deny the Disease of Life <a href="#" onclick="yt.www.watch.player.seekTo(53*60+07);return false;">53:07</a><br /><br />More info/Buy: http://crepusculonegro.storenvy.com/products/257645-cn-03-arizmenda-within-the-vacuum-of-infinity<br /><br />No copyright is intended. The rights to this video are assumed by the owner and its affiliates.',
-            4009,
-            [{
-                'start_time': 0,
-                'end_time': 707,
-                'title': '1. Those Beaten Paths of Confusion',
-            }, {
-                'start_time': 707,
-                'end_time': 1590,
-                'title': '2. Beyond the Shadows of Emptiness & Nothingness',
-            }, {
-                'start_time': 1590,
-                'end_time': 2157,
-                'title': '3. Poison Yourself...With Thought',
-            }, {
-                'start_time': 2157,
-                'end_time': 2672,
-                'title': '4. The Agents of Transformation',
-            }, {
-                'start_time': 2672,
-                'end_time': 3187,
-                'title': '5. Drowning in the Pain of Consciousness',
-            }, {
-                'start_time': 3187,
-                'end_time': 4009,
-                'title': '6. Deny the Disease of Life',
-            }]
-        ),
-        (
-            # https://www.youtube.com/watch?v=WjL4pSzog9w
-            # pattern: 00:00 <title>
-            '<a href="https://arizmenda.bandcamp.com/merch/despairs-depths-descended-cd" class="yt-uix-servicelink  " data-target-new-window="True" data-servicelink="CDAQ6TgYACITCNf1raqT2dMCFdRjGAod_o0CBSj4HQ" data-url="https://arizmenda.bandcamp.com/merch/despairs-depths-descended-cd" rel="nofollow noopener" target="_blank">https://arizmenda.bandcamp.com/merch/...</a><br /><br /><a href="#" onclick="yt.www.watch.player.seekTo(00*60+00);return false;">00:00</a> Christening Unborn Deformities <br /><a href="#" onclick="yt.www.watch.player.seekTo(07*60+08);return false;">07:08</a> Taste of Purity<br /><a href="#" onclick="yt.www.watch.player.seekTo(16*60+16);return false;">16:16</a> Sculpting Sins of a Universal Tongue<br /><a href="#" onclick="yt.www.watch.player.seekTo(24*60+45);return false;">24:45</a> Birth<br /><a href="#" onclick="yt.www.watch.player.seekTo(31*60+24);return false;">31:24</a> Neves<br /><a href="#" onclick="yt.www.watch.player.seekTo(37*60+55);return false;">37:55</a> Libations in Limbo',
-            2705,
-            [{
-                'start_time': 0,
-                'end_time': 428,
-                'title': 'Christening Unborn Deformities',
-            }, {
-                'start_time': 428,
-                'end_time': 976,
-                'title': 'Taste of Purity',
-            }, {
-                'start_time': 976,
-                'end_time': 1485,
-                'title': 'Sculpting Sins of a Universal Tongue',
-            }, {
-                'start_time': 1485,
-                'end_time': 1884,
-                'title': 'Birth',
-            }, {
-                'start_time': 1884,
-                'end_time': 2275,
-                'title': 'Neves',
-            }, {
-                'start_time': 2275,
-                'end_time': 2705,
-                'title': 'Libations in Limbo',
-            }]
-        ),
-        (
-            # https://www.youtube.com/watch?v=o3r1sn-t3is
-            # pattern: <title> 00:00 <note>
-            'Download this show in MP3: <a href="http://sh.st/njZKK" class="yt-uix-servicelink  " data-url="http://sh.st/njZKK" data-target-new-window="True" data-servicelink="CDAQ6TgYACITCK3j8_6o2dMCFVDCGAoduVAKKij4HQ" rel="nofollow noopener" target="_blank">http://sh.st/njZKK</a><br /><br />Setlist:<br />I-E-A-I-A-I-O <a href="#" onclick="yt.www.watch.player.seekTo(00*60+45);return false;">00:45</a><br />Suite-Pee <a href="#" onclick="yt.www.watch.player.seekTo(4*60+26);return false;">4:26</a>  (Incomplete)<br />Attack <a href="#" onclick="yt.www.watch.player.seekTo(5*60+31);return false;">5:31</a> (First live performance since 2011)<br />Prison Song <a href="#" onclick="yt.www.watch.player.seekTo(8*60+42);return false;">8:42</a><br />Know <a href="#" onclick="yt.www.watch.player.seekTo(12*60+32);return false;">12:32</a> (First live performance since 2011)<br />Aerials <a href="#" onclick="yt.www.watch.player.seekTo(15*60+32);return false;">15:32</a><br />Soldier Side - Intro <a href="#" onclick="yt.www.watch.player.seekTo(19*60+13);return false;">19:13</a><br />B.Y.O.B. <a href="#" onclick="yt.www.watch.player.seekTo(20*60+09);return false;">20:09</a><br />Soil <a href="#" onclick="yt.www.watch.player.seekTo(24*60+32);return false;">24:32</a><br />Darts <a href="#" onclick="yt.www.watch.player.seekTo(27*60+48);return false;">27:48</a><br />Radio/Video <a href="#" onclick="yt.www.watch.player.seekTo(30*60+38);return false;">30:38</a><br />Hypnotize <a href="#" onclick="yt.www.watch.player.seekTo(35*60+05);return false;">35:05</a><br />Temper <a href="#" onclick="yt.www.watch.player.seekTo(38*60+08);return false;">38:08</a> (First live performance since 1999)<br />CUBErt <a href="#" onclick="yt.www.watch.player.seekTo(41*60+00);return false;">41:00</a><br />Needles <a href="#" onclick="yt.www.watch.player.seekTo(42*60+57);return false;">42:57</a><br />Deer Dance <a href="#" onclick="yt.www.watch.player.seekTo(46*60+27);return false;">46:27</a><br />Bounce <a href="#" onclick="yt.www.watch.player.seekTo(49*60+38);return false;">49:38</a><br />Suggestions <a href="#" onclick="yt.www.watch.player.seekTo(51*60+25);return false;">51:25</a><br />Psycho <a href="#" onclick="yt.www.watch.player.seekTo(53*60+52);return false;">53:52</a><br />Chop Suey! <a href="#" onclick="yt.www.watch.player.seekTo(58*60+13);return false;">58:13</a><br />Lonely Day <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+01*60+15);return false;">1:01:15</a><br />Question! <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+04*60+14);return false;">1:04:14</a><br />Lost in Hollywood <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+08*60+10);return false;">1:08:10</a><br />Vicinity of Obscenity  <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+13*60+40);return false;">1:13:40</a>(First live performance since 2012)<br />Forest <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+16*60+17);return false;">1:16:17</a><br />Cigaro <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+20*60+02);return false;">1:20:02</a><br />Toxicity <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+23*60+57);return false;">1:23:57</a>(with Chino Moreno)<br />Sugar <a href="#" onclick="yt.www.watch.player.seekTo(1*3600+27*60+53);return false;">1:27:53</a>',
-            5640,
-            [{
-                'start_time': 45,
-                'end_time': 266,
-                'title': 'I-E-A-I-A-I-O',
-            }, {
-                'start_time': 266,
-                'end_time': 331,
-                'title': 'Suite-Pee (Incomplete)',
-            }, {
-                'start_time': 331,
-                'end_time': 522,
-                'title': 'Attack (First live performance since 2011)',
-            }, {
-                'start_time': 522,
-                'end_time': 752,
-                'title': 'Prison Song',
-            }, {
-                'start_time': 752,
-                'end_time': 932,
-                'title': 'Know (First live performance since 2011)',
-            }, {
-                'start_time': 932,
-                'end_time': 1153,
-                'title': 'Aerials',
-            }, {
-                'start_time': 1153,
-                'end_time': 1209,
-                'title': 'Soldier Side - Intro',
-            }, {
-                'start_time': 1209,
-                'end_time': 1472,
-                'title': 'B.Y.O.B.',
-            }, {
-                'start_time': 1472,
-                'end_time': 1668,
-                'title': 'Soil',
-            }, {
-                'start_time': 1668,
-                'end_time': 1838,
-                'title': 'Darts',
-            }, {
-                'start_time': 1838,
-                'end_time': 2105,
-                'title': 'Radio/Video',
-            }, {
-                'start_time': 2105,
-                'end_time': 2288,
-                'title': 'Hypnotize',
-            }, {
-                'start_time': 2288,
-                'end_time': 2460,
-                'title': 'Temper (First live performance since 1999)',
-            }, {
-                'start_time': 2460,
-                'end_time': 2577,
-                'title': 'CUBErt',
-            }, {
-                'start_time': 2577,
-                'end_time': 2787,
-                'title': 'Needles',
-            }, {
-                'start_time': 2787,
-                'end_time': 2978,
-                'title': 'Deer Dance',
-            }, {
-                'start_time': 2978,
-                'end_time': 3085,
-                'title': 'Bounce',
-            }, {
-                'start_time': 3085,
-                'end_time': 3232,
-                'title': 'Suggestions',
-            }, {
-                'start_time': 3232,
-                'end_time': 3493,
-                'title': 'Psycho',
-            }, {
-                'start_time': 3493,
-                'end_time': 3675,
-                'title': 'Chop Suey!',
-            }, {
-                'start_time': 3675,
-                'end_time': 3854,
-                'title': 'Lonely Day',
-            }, {
-                'start_time': 3854,
-                'end_time': 4090,
-                'title': 'Question!',
-            }, {
-                'start_time': 4090,
-                'end_time': 4420,
-                'title': 'Lost in Hollywood',
-            }, {
-                'start_time': 4420,
-                'end_time': 4577,
-                'title': 'Vicinity of Obscenity (First live performance since 2012)',
-            }, {
-                'start_time': 4577,
-                'end_time': 4802,
-                'title': 'Forest',
-            }, {
-                'start_time': 4802,
-                'end_time': 5037,
-                'title': 'Cigaro',
-            }, {
-                'start_time': 5037,
-                'end_time': 5273,
-                'title': 'Toxicity (with Chino Moreno)',
-            }, {
-                'start_time': 5273,
-                'end_time': 5640,
-                'title': 'Sugar',
-            }]
-        ),
-        (
-            # https://www.youtube.com/watch?v=PkYLQbsqCE8
-            # pattern: <num> - <title> [<latinized title>] 0:00:00
-            '''Затемно (Zatemno) is an Obscure Black Metal Band from Russia.<br /><br />"Во прах (Vo prakh)'' Into The Ashes", Debut mini-album released may 6, 2016, by Death Knell Productions<br />Released on 6 panel digipak CD, limited to 100 copies only<br />And digital format on Bandcamp<br /><br />Tracklist<br /><br />1 - Во прах [Vo prakh] <a href="#" onclick="yt.www.watch.player.seekTo(0*3600+00*60+00);return false;">0:00:00</a><br />2 - Искупление [Iskupleniye] <a href="#" onclick="yt.www.watch.player.seekTo(0*3600+08*60+10);return false;">0:08:10</a><br />3 - Из серпов луны...[Iz serpov luny] <a href="#" onclick="yt.www.watch.player.seekTo(0*3600+14*60+30);return false;">0:14:30</a><br /><br />Links:<br /><a href="https://deathknellprod.bandcamp.com/album/--2" class="yt-uix-servicelink  " data-target-new-window="True" data-url="https://deathknellprod.bandcamp.com/album/--2" data-servicelink="CC8Q6TgYACITCNP234Kr2dMCFcNxGAodQqsIwSj4HQ" target="_blank" rel="nofollow noopener">https://deathknellprod.bandcamp.com/a...</a><br /><a href="https://www.facebook.com/DeathKnellProd/" class="yt-uix-servicelink  " data-target-new-window="True" data-url="https://www.facebook.com/DeathKnellProd/" data-servicelink="CC8Q6TgYACITCNP234Kr2dMCFcNxGAodQqsIwSj4HQ" target="_blank" rel="nofollow noopener">https://www.facebook.com/DeathKnellProd/</a><br /><br /><br />I don't have any right about this artifact, my only intention is to spread the music of the band, all rights are reserved to the Затемно (Zatemno) and his producers, Death Knell Productions.<br /><br />------------------------------------------------------------------<br /><br />Subscribe for more videos like this.<br />My link: <a href="https://web.facebook.com/AttackOfTheDragons" class="yt-uix-servicelink  " data-target-new-window="True" data-url="https://web.facebook.com/AttackOfTheDragons" data-servicelink="CC8Q6TgYACITCNP234Kr2dMCFcNxGAodQqsIwSj4HQ" target="_blank" rel="nofollow noopener">https://web.facebook.com/AttackOfTheD...</a>''',
-            1138,
-            [{
-                'start_time': 0,
-                'end_time': 490,
-                'title': '1 - Во прах [Vo prakh]',
-            }, {
-                'start_time': 490,
-                'end_time': 870,
-                'title': '2 - Искупление [Iskupleniye]',
-            }, {
-                'start_time': 870,
-                'end_time': 1138,
-                'title': '3 - Из серпов луны...[Iz serpov luny]',
-            }]
-        ),
-        (
-            # https://www.youtube.com/watch?v=xZW70zEasOk
-            # time point more than duration
-            '''● LCS Spring finals: Saturday and Sunday from <a href="#" onclick="yt.www.watch.player.seekTo(13*60+30);return false;">13:30</a> outside the venue! <br />● PAX East: Fri, Sat & Sun - more info in tomorrows video on the main channel!''',
-            283,
-            []
-        ),
-    ]
-
-    def test_youtube_chapters(self):
-        for description, duration, expected_chapters in self._TEST_CASES:
-            ie = YoutubeIE()
-            expect_value(
-                self, ie._extract_chapters_from_description(description, duration),
-                expected_chapters, None)
-
-
-if __name__ == '__main__':
-    unittest.main()
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5f6769878..edaca0658 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1223,6 +1223,46 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         video_id = mobj.group(2)
         return video_id
 
+    def _extract_chapters_from_json(self, data, video_id, duration):
+        chapters_list = try_get(
+            data,
+            lambda x: x['playerOverlays']
+                       ['playerOverlayRenderer']
+                       ['decoratedPlayerBarRenderer']
+                       ['decoratedPlayerBarRenderer']
+                       ['playerBar']
+                       ['chapteredPlayerBarRenderer']
+                       ['chapters'],
+            list)
+        if not chapters_list:
+            return
+
+        def chapter_time(chapter):
+            return float_or_none(
+                try_get(
+                    chapter,
+                    lambda x: x['chapterRenderer']['timeRangeStartMillis'],
+                    int),
+                scale=1000)
+        chapters = []
+        for next_num, chapter in enumerate(chapters_list, start=1):
+            start_time = chapter_time(chapter)
+            if start_time is None:
+                continue
+            end_time = (chapter_time(chapters_list[next_num])
+                        if next_num < len(chapters_list) else duration)
+            if end_time is None:
+                continue
+            title = try_get(
+                chapter, lambda x: x['chapterRenderer']['title']['simpleText'],
+                compat_str)
+            chapters.append({
+                'start_time': start_time,
+                'end_time': end_time,
+                'title': title,
+            })
+        return chapters
+
     def _extract_yt_initial_variable(self, webpage, regex, video_id, name):
         return self._parse_json(self._search_regex(
             (r'%s\s*%s' % (regex, self._YT_INITIAL_BOUNDARY_RE),
@@ -1597,31 +1637,34 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'next', {'videoId': video_id}, video_id, fatal=False)
 
         if initial_data:
-            for engagment_pannel in (initial_data.get('engagementPanels') or []):
-                contents = try_get(
-                    engagment_pannel, lambda x: x['engagementPanelSectionListRenderer']['content']['macroMarkersListRenderer']['contents'],
-                    list)
-                if not contents:
-                    continue
-
-                def chapter_time(mmlir):
-                    return parse_duration(mmlir.get(
-                        get_text(mmlir.get('timeDescription'))))
-
-                chapters = []
-                for next_num, content in enumerate(contents, start=1):
-                    mmlir = content.get('macroMarkersListItemRenderer') or {}
-                    start_time = chapter_time(mmlir)
-                    end_time = chapter_time(try_get(
-                        contents, lambda x: x[next_num]['macroMarkersListItemRenderer'])) \
-                        if next_num < len(contents) else duration
-                    if not (start_time and end_time):
+            chapters = self._extract_chapters_from_json(
+                initial_data, video_id, duration)
+            if not chapters:
+                for engagment_pannel in (initial_data.get('engagementPanels') or []):
+                    contents = try_get(
+                        engagment_pannel, lambda x: x['engagementPanelSectionListRenderer']['content']['macroMarkersListRenderer']['contents'],
+                        list)
+                    if not contents:
                         continue
-                    chapters.append({
-                        'start_time': start_time,
-                        'end_time': end_time,
-                        'title': get_text(mmlir.get('title')),
-                    })
+
+                    def chapter_time(mmlir):
+                        return parse_duration(mmlir.get(
+                            get_text(mmlir.get('timeDescription'))))
+
+                    for next_num, content in enumerate(contents, start=1):
+                        mmlir = content.get('macroMarkersListItemRenderer') or {}
+                        start_time = chapter_time(mmlir)
+                        end_time = chapter_time(try_get(
+                            contents, lambda x: x[next_num]['macroMarkersListItemRenderer'])) \
+                            if next_num < len(contents) else duration
+                        if not (start_time and end_time):
+                            continue
+                        chapters.append({
+                            'start_time': start_time,
+                            'end_time': end_time,
+                            'title': get_text(mmlir.get('title')),
+                        })
+            if chapters:
                 info['chapters'] = chapters
 
             contents = try_get(

From b46483a6ec6a42889fc16d53afd76d147748785f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 1 Feb 2021 16:35:07 +0100
Subject: [PATCH 171/860] [youtube/test_youtube_signature] fix test

---
 test/test_youtube_signature.py | 6 +-----
 1 file changed, 1 insertion(+), 5 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 69df30eda..b5a4d0d5f 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -86,13 +86,9 @@ class TestPlayerInfo(unittest.TestCase):
             ('https://www.youtube.com/yts/jsbin/player-en_US-vflaxXRn1/base.js', 'vflaxXRn1'),
             ('https://s.ytimg.com/yts/jsbin/html5player-en_US-vflXGBaUN.js', 'vflXGBaUN'),
             ('https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js', 'vflKjOTVq'),
-            ('http://s.ytimg.com/yt/swfbin/watch_as3-vflrEm9Nq.swf', 'vflrEm9Nq'),
-            ('https://s.ytimg.com/yts/swfbin/player-vflenCdZL/watch_as3.swf', 'vflenCdZL'),
         )
         for player_url, expected_player_id in PLAYER_URLS:
-            expected_player_type = player_url.split('.')[-1]
-            player_type, player_id = YoutubeIE._extract_player_info(player_url)
-            self.assertEqual(player_type, expected_player_type)
+            player_id = YoutubeIE._extract_player_info(player_url)
             self.assertEqual(player_id, expected_player_id)
 
 

From 159a3d48dfb2b4ed77dc691433e420506c9340c3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 1 Feb 2021 16:36:19 +0100
Subject: [PATCH 172/860] [youtube] keep _formats array for format sorting
 tests

---
 youtube_dl/extractor/youtube.py | 105 ++++++++++++++++++++++++++++++++
 1 file changed, 105 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index edaca0658..ed844e2a3 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1019,6 +1019,111 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'only_matching': True,
         },
     ]
+    _formats = {
+        '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
+        '6': {'ext': 'flv', 'width': 450, 'height': 270, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
+        '13': {'ext': '3gp', 'acodec': 'aac', 'vcodec': 'mp4v'},
+        '17': {'ext': '3gp', 'width': 176, 'height': 144, 'acodec': 'aac', 'abr': 24, 'vcodec': 'mp4v'},
+        '18': {'ext': 'mp4', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 96, 'vcodec': 'h264'},
+        '22': {'ext': 'mp4', 'width': 1280, 'height': 720, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
+        '34': {'ext': 'flv', 'width': 640, 'height': 360, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
+        '35': {'ext': 'flv', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
+        # itag 36 videos are either 320x180 (BaW_jenozKc) or 320x240 (__2ABJjxzNo), abr varies as well
+        '36': {'ext': '3gp', 'width': 320, 'acodec': 'aac', 'vcodec': 'mp4v'},
+        '37': {'ext': 'mp4', 'width': 1920, 'height': 1080, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
+        '38': {'ext': 'mp4', 'width': 4096, 'height': 3072, 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264'},
+        '43': {'ext': 'webm', 'width': 640, 'height': 360, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
+        '44': {'ext': 'webm', 'width': 854, 'height': 480, 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8'},
+        '45': {'ext': 'webm', 'width': 1280, 'height': 720, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
+        '46': {'ext': 'webm', 'width': 1920, 'height': 1080, 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8'},
+        '59': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
+        '78': {'ext': 'mp4', 'width': 854, 'height': 480, 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264'},
+
+
+        # 3D videos
+        '82': {'ext': 'mp4', 'height': 360, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
+        '83': {'ext': 'mp4', 'height': 480, 'format_note': '3D', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -20},
+        '84': {'ext': 'mp4', 'height': 720, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
+        '85': {'ext': 'mp4', 'height': 1080, 'format_note': '3D', 'acodec': 'aac', 'abr': 192, 'vcodec': 'h264', 'preference': -20},
+        '100': {'ext': 'webm', 'height': 360, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 128, 'vcodec': 'vp8', 'preference': -20},
+        '101': {'ext': 'webm', 'height': 480, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
+        '102': {'ext': 'webm', 'height': 720, 'format_note': '3D', 'acodec': 'vorbis', 'abr': 192, 'vcodec': 'vp8', 'preference': -20},
+
+        # Apple HTTP Live Streaming
+        '91': {'ext': 'mp4', 'height': 144, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
+        '92': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
+        '93': {'ext': 'mp4', 'height': 360, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
+        '94': {'ext': 'mp4', 'height': 480, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 128, 'vcodec': 'h264', 'preference': -10},
+        '95': {'ext': 'mp4', 'height': 720, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
+        '96': {'ext': 'mp4', 'height': 1080, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 256, 'vcodec': 'h264', 'preference': -10},
+        '132': {'ext': 'mp4', 'height': 240, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 48, 'vcodec': 'h264', 'preference': -10},
+        '151': {'ext': 'mp4', 'height': 72, 'format_note': 'HLS', 'acodec': 'aac', 'abr': 24, 'vcodec': 'h264', 'preference': -10},
+
+        # DASH mp4 video
+        '133': {'ext': 'mp4', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '134': {'ext': 'mp4', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '135': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '136': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '137': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '138': {'ext': 'mp4', 'format_note': 'DASH video', 'vcodec': 'h264'},  # Height can vary (https://github.com/ytdl-org/youtube-dl/issues/4559)
+        '160': {'ext': 'mp4', 'height': 144, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '212': {'ext': 'mp4', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '264': {'ext': 'mp4', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'h264'},
+        '298': {'ext': 'mp4', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
+        '299': {'ext': 'mp4', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'h264', 'fps': 60},
+        '266': {'ext': 'mp4', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'h264'},
+
+        # Dash mp4 audio
+        '139': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 48, 'container': 'm4a_dash'},
+        '140': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 128, 'container': 'm4a_dash'},
+        '141': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'abr': 256, 'container': 'm4a_dash'},
+        '256': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
+        '258': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'aac', 'container': 'm4a_dash'},
+        '325': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'dtse', 'container': 'm4a_dash'},
+        '328': {'ext': 'm4a', 'format_note': 'DASH audio', 'acodec': 'ec-3', 'container': 'm4a_dash'},
+
+        # Dash webm
+        '167': {'ext': 'webm', 'height': 360, 'width': 640, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
+        '168': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
+        '169': {'ext': 'webm', 'height': 720, 'width': 1280, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
+        '170': {'ext': 'webm', 'height': 1080, 'width': 1920, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
+        '218': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
+        '219': {'ext': 'webm', 'height': 480, 'width': 854, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp8'},
+        '278': {'ext': 'webm', 'height': 144, 'format_note': 'DASH video', 'container': 'webm', 'vcodec': 'vp9'},
+        '242': {'ext': 'webm', 'height': 240, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '243': {'ext': 'webm', 'height': 360, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '244': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '245': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '246': {'ext': 'webm', 'height': 480, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '247': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '248': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '271': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        # itag 272 videos are either 3840x2160 (e.g. RtoitU2A-3E) or 7680x4320 (sLprVF6d7Ug)
+        '272': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '302': {'ext': 'webm', 'height': 720, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
+        '303': {'ext': 'webm', 'height': 1080, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
+        '308': {'ext': 'webm', 'height': 1440, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
+        '313': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9'},
+        '315': {'ext': 'webm', 'height': 2160, 'format_note': 'DASH video', 'vcodec': 'vp9', 'fps': 60},
+
+        # Dash webm audio
+        '171': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 128},
+        '172': {'ext': 'webm', 'acodec': 'vorbis', 'format_note': 'DASH audio', 'abr': 256},
+
+        # Dash webm audio with opus inside
+        '249': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 50},
+        '250': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 70},
+        '251': {'ext': 'webm', 'format_note': 'DASH audio', 'acodec': 'opus', 'abr': 160},
+
+        # RTMP (unnamed)
+        '_rtmp': {'protocol': 'rtmp'},
+
+        # av01 video only formats sometimes served with "unknown" codecs
+        '394': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
+        '395': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
+        '396': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
+        '397': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
+    }
 
     def __init__(self, *args, **kwargs):
         super(YoutubeIE, self).__init__(*args, **kwargs)

From efef4ddf51c375c3a9eb12355a61a21d69aec33f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 1 Feb 2021 16:49:52 +0100
Subject: [PATCH 173/860] [youtube] fix chapter extraction fallback

---
 youtube_dl/extractor/youtube.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ed844e2a3..65fa777e4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1753,22 +1753,25 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         continue
 
                     def chapter_time(mmlir):
-                        return parse_duration(mmlir.get(
-                            get_text(mmlir.get('timeDescription'))))
+                        return parse_duration(
+                            get_text(mmlir.get('timeDescription')))
 
+                    chapters = []
                     for next_num, content in enumerate(contents, start=1):
                         mmlir = content.get('macroMarkersListItemRenderer') or {}
                         start_time = chapter_time(mmlir)
                         end_time = chapter_time(try_get(
                             contents, lambda x: x[next_num]['macroMarkersListItemRenderer'])) \
                             if next_num < len(contents) else duration
-                        if not (start_time and end_time):
+                        if start_time is None or end_time is None:
                             continue
                         chapters.append({
                             'start_time': start_time,
                             'end_time': end_time,
                             'title': get_text(mmlir.get('title')),
                         })
+                    if chapters:
+                        break
             if chapters:
                 info['chapters'] = chapters
 

From 65eee5a745f705a7904709accdba47efb852cc6a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 1 Feb 2021 18:12:35 +0100
Subject: [PATCH 174/860] [youtube] improve subtitle extraction

---
 youtube_dl/extractor/youtube.py | 43 ++++++++++++++-------------------
 1 file changed, 18 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 65fa777e4..75a007353 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1664,7 +1664,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             player_response,
             lambda x: x['captions']['playerCaptionsTracklistRenderer'], dict)
         if pctr:
-            def process_language(container, base_url, caption, query):
+            def process_language(container, base_url, lang_code, query):
                 lang_subs = []
                 for fmt in self._SUBTITLE_FORMATS:
                     query.update({
@@ -1674,35 +1674,28 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         'ext': fmt,
                         'url': update_url_query(base_url, query),
                     })
-                subtitles[caption['languageCode']] = lang_subs
+                container[lang_code] = lang_subs
 
             subtitles = {}
-            for caption_track in pctr['captionTracks']:
-                base_url = caption_track['baseUrl']
+            for caption_track in (pctr.get('captionTracks') or []):
+                base_url = caption_track.get('baseUrl')
+                if not base_url:
+                    continue
                 if caption_track.get('kind') != 'asr':
-                    lang_subs = []
-                    for fmt in self._SUBTITLE_FORMATS:
-                        lang_subs.append({
-                            'ext': fmt,
-                            'url': update_url_query(base_url, {
-                                'fmt': fmt,
-                            }),
-                        })
-                    subtitles[caption_track['languageCode']] = lang_subs
+                    lang_code = caption_track.get('languageCode')
+                    if not lang_code:
+                        continue
+                    process_language(
+                        subtitles, base_url, lang_code, {})
                     continue
                 automatic_captions = {}
-                for translation_language in pctr['translationLanguages']:
-                    translation_language_code = translation_language['languageCode']
-                    lang_subs = []
-                    for fmt in self._SUBTITLE_FORMATS:
-                        lang_subs.append({
-                            'ext': fmt,
-                            'url': update_url_query(base_url, {
-                                'fmt': fmt,
-                                'tlang': translation_language_code,
-                            }),
-                        })
-                    automatic_captions[translation_language_code] = lang_subs
+                for translation_language in (pctr.get('translationLanguages') or []):
+                    translation_language_code = translation_language.get('languageCode')
+                    if not translation_language_code:
+                        continue
+                    process_language(
+                        automatic_captions, base_url, translation_language_code,
+                        {'tlang': translation_language_code})
                 info['automatic_captions'] = automatic_captions
             info['subtitles'] = subtitles
 

From 8fa7cc387d699899114f7430bcf61837d58557a8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 1 Feb 2021 21:35:18 +0100
Subject: [PATCH 175/860] [vidio] improve metadata extraction

---
 youtube_dl/extractor/vidio.py | 86 ++++++++++++++++++++---------------
 1 file changed, 49 insertions(+), 37 deletions(-)

diff --git a/youtube_dl/extractor/vidio.py b/youtube_dl/extractor/vidio.py
index b48baf00b..b1243e847 100644
--- a/youtube_dl/extractor/vidio.py
+++ b/youtube_dl/extractor/vidio.py
@@ -4,7 +4,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import int_or_none
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    str_or_none,
+    strip_or_none,
+    try_get,
+)
 
 
 class VidioIE(InfoExtractor):
@@ -21,57 +27,63 @@ class VidioIE(InfoExtractor):
             'thumbnail': r're:^https?://.*\.jpg$',
             'duration': 149,
             'like_count': int,
+            'uploader': 'TWELVE Pic',
+            'timestamp': 1444902800,
+            'upload_date': '20151015',
+            'uploader_id': 'twelvepictures',
+            'channel': 'Cover Music Video',
+            'channel_id': '280236',
+            'view_count': int,
+            'dislike_count': int,
+            'comment_count': int,
+            'tags': 'count:4',
         },
     }, {
         'url': 'https://www.vidio.com/watch/77949-south-korea-test-fires-missile-that-can-strike-all-of-the-north',
         'only_matching': True,
     }]
 
+    def _real_initialize(self):
+        self._api_key = self._download_json(
+            'https://www.vidio.com/auth', None, data=b'')['api_key']
+
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id, display_id = mobj.group('id', 'display_id')
+        video_id, display_id = re.match(self._VALID_URL, url).groups()
+        data = self._download_json(
+            'https://api.vidio.com/videos/' + video_id, display_id, headers={
+                'Content-Type': 'application/vnd.api+json',
+                'X-API-KEY': self._api_key,
+            })
+        video = data['videos'][0]
+        title = video['title'].strip()
 
-        webpage = self._download_webpage(url, display_id)
-
-        title = self._og_search_title(webpage)
-
-        m3u8_url, duration, thumbnail = [None] * 3
-
-        clips = self._parse_json(
-            self._html_search_regex(
-                r'data-json-clips\s*=\s*(["\'])(?P<data>\[.+?\])\1',
-                webpage, 'video data', default='[]', group='data'),
-            display_id, fatal=False)
-        if clips:
-            clip = clips[0]
-            m3u8_url = clip.get('sources', [{}])[0].get('file')
-            duration = clip.get('clip_duration')
-            thumbnail = clip.get('image')
-
-        m3u8_url = m3u8_url or self._search_regex(
-            r'data(?:-vjs)?-clip-hls-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
-            webpage, 'hls url', group='url')
         formats = self._extract_m3u8_formats(
-            m3u8_url, display_id, 'mp4', entry_protocol='m3u8_native')
+            data['clips'][0]['hls_url'], display_id, 'mp4', 'm3u8_native')
         self._sort_formats(formats)
 
-        duration = int_or_none(duration or self._search_regex(
-            r'data-video-duration=(["\'])(?P<duration>\d+)\1', webpage,
-            'duration', fatal=False, group='duration'))
-        thumbnail = thumbnail or self._og_search_thumbnail(webpage)
-
-        like_count = int_or_none(self._search_regex(
-            (r'<span[^>]+data-comment-vote-count=["\'](\d+)',
-             r'<span[^>]+class=["\'].*?\blike(?:__|-)count\b.*?["\'][^>]*>\s*(\d+)'),
-            webpage, 'like count', fatal=False))
+        get_first = lambda x: try_get(data, lambda y: y[x + 's'][0], dict) or {}
+        channel = get_first('channel')
+        user = get_first('user')
+        username = user.get('username')
+        get_count = lambda x: int_or_none(video.get('total_' + x))
 
         return {
             'id': video_id,
             'display_id': display_id,
             'title': title,
-            'description': self._og_search_description(webpage),
-            'thumbnail': thumbnail,
-            'duration': duration,
-            'like_count': like_count,
+            'description': strip_or_none(video.get('description')),
+            'thumbnail': video.get('image_url_medium'),
+            'duration': int_or_none(video.get('duration')),
+            'like_count': get_count('likes'),
             'formats': formats,
+            'uploader': user.get('name'),
+            'timestamp': parse_iso8601(video.get('created_at')),
+            'uploader_id': username,
+            'uploader_url': 'https://www.vidio.com/@' + username if username else None,
+            'channel': channel.get('name'),
+            'channel_id': str_or_none(channel.get('id')),
+            'view_count': get_count('view_count'),
+            'dislike_count': get_count('dislikes'),
+            'comment_count': get_count('comments'),
+            'tags': video.get('tag_list'),
         }

From c11f7cf9bd6ef239f25e7fb9c54e092ae1490e2d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 1 Feb 2021 22:35:28 +0100
Subject: [PATCH 176/860] [vidzi] remove extractor(closes #12629)

---
 youtube_dl/extractor/extractors.py |  1 -
 youtube_dl/extractor/vidzi.py      | 68 ------------------------------
 2 files changed, 69 deletions(-)
 delete mode 100644 youtube_dl/extractor/vidzi.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index ab8d6a5a5..97b0b4034 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1399,7 +1399,6 @@ from .vidme import (
     VidmeUserIE,
     VidmeUserLikesIE,
 )
-from .vidzi import VidziIE
 from .vier import VierIE, VierVideosIE
 from .viewlift import (
     ViewLiftIE,
diff --git a/youtube_dl/extractor/vidzi.py b/youtube_dl/extractor/vidzi.py
deleted file mode 100644
index 42ea4952c..000000000
--- a/youtube_dl/extractor/vidzi.py
+++ /dev/null
@@ -1,68 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    decode_packed_codes,
-    js_to_json,
-    NO_DEFAULT,
-    PACKED_CODES_RE,
-)
-
-
-class VidziIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?vidzi\.(?:tv|cc|si|nu)/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
-    _TESTS = [{
-        'url': 'http://vidzi.tv/cghql9yq6emu.html',
-        'md5': '4f16c71ca0c8c8635ab6932b5f3f1660',
-        'info_dict': {
-            'id': 'cghql9yq6emu',
-            'ext': 'mp4',
-            'title': 'youtube-dl test video  1\\\\2\'3/4<5\\\\6ä7↭',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }, {
-        'url': 'http://vidzi.tv/embed-4z2yb0rzphe9-600x338.html',
-        'only_matching': True,
-    }, {
-        'url': 'http://vidzi.cc/cghql9yq6emu.html',
-        'only_matching': True,
-    }, {
-        'url': 'https://vidzi.si/rph9gztxj1et.html',
-        'only_matching': True,
-    }, {
-        'url': 'http://vidzi.nu/cghql9yq6emu.html',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(
-            'http://vidzi.tv/%s' % video_id, video_id)
-        title = self._html_search_regex(
-            r'(?s)<h2 class="video-title">(.*?)</h2>', webpage, 'title')
-
-        codes = [webpage]
-        codes.extend([
-            decode_packed_codes(mobj.group(0)).replace('\\\'', '\'')
-            for mobj in re.finditer(PACKED_CODES_RE, webpage)])
-        for num, code in enumerate(codes, 1):
-            jwplayer_data = self._parse_json(
-                self._search_regex(
-                    r'setup\(([^)]+)\)', code, 'jwplayer data',
-                    default=NO_DEFAULT if num == len(codes) else '{}'),
-                video_id, transform_source=lambda s: js_to_json(
-                    re.sub(r'\s*\+\s*window\[.+?\]', '', s)))
-            if jwplayer_data:
-                break
-
-        info_dict = self._parse_jwplayer_data(jwplayer_data, video_id, require_title=False)
-        info_dict['title'] = title
-
-        return info_dict

From 0e3a9684795c6c53546dace9e917ed11c4ae72a5 Mon Sep 17 00:00:00 2001
From: Viren Rajput <virendra.rajput567@gmail.com>
Date: Mon, 1 Feb 2021 04:56:33 +0000
Subject: [PATCH 177/860] [egghead] update API domain(closes #28038)

---
 youtube_dl/extractor/egghead.py | 29 +++++++++++++++++------------
 1 file changed, 17 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/egghead.py b/youtube_dl/extractor/egghead.py
index df11dc206..94dd75b9b 100644
--- a/youtube_dl/extractor/egghead.py
+++ b/youtube_dl/extractor/egghead.py
@@ -12,7 +12,14 @@ from ..utils import (
 )
 
 
-class EggheadCourseIE(InfoExtractor):
+class EggheadBaseIE(InfoExtractor):
+    def _call_api(self, path, video_id, resource, fatal=True):
+        return self._download_json(
+            'https://app.egghead.io/api/v1/' + path,
+            video_id, 'Downloading %s JSON' % resource)
+
+
+class EggheadCourseIE(EggheadBaseIE):
     IE_DESC = 'egghead.io course'
     IE_NAME = 'egghead:course'
     _VALID_URL = r'https://egghead\.io/courses/(?P<id>[^/?#&]+)'
@@ -28,10 +35,9 @@ class EggheadCourseIE(InfoExtractor):
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
-
-        lessons = self._download_json(
-            'https://egghead.io/api/v1/series/%s/lessons' % playlist_id,
-            playlist_id, 'Downloading course lessons JSON')
+        series_path = 'series/' + playlist_id
+        lessons = self._call_api(
+            series_path + '/lessons', playlist_id, 'course lessons')
 
         entries = []
         for lesson in lessons:
@@ -44,9 +50,8 @@ class EggheadCourseIE(InfoExtractor):
             entries.append(self.url_result(
                 lesson_url, ie=EggheadLessonIE.ie_key(), video_id=lesson_id))
 
-        course = self._download_json(
-            'https://egghead.io/api/v1/series/%s' % playlist_id,
-            playlist_id, 'Downloading course JSON', fatal=False) or {}
+        course = self._call_api(
+            series_path, playlist_id, 'course', False) or {}
 
         playlist_id = course.get('id')
         if playlist_id:
@@ -57,7 +62,7 @@ class EggheadCourseIE(InfoExtractor):
             course.get('description'))
 
 
-class EggheadLessonIE(InfoExtractor):
+class EggheadLessonIE(EggheadBaseIE):
     IE_DESC = 'egghead.io lesson'
     IE_NAME = 'egghead:lesson'
     _VALID_URL = r'https://egghead\.io/(?:api/v1/)?lessons/(?P<id>[^/?#&]+)'
@@ -74,7 +79,7 @@ class EggheadLessonIE(InfoExtractor):
             'upload_date': '20161209',
             'duration': 304,
             'view_count': 0,
-            'tags': ['javascript', 'free'],
+            'tags': ['free', 'javascript'],
         },
         'params': {
             'skip_download': True,
@@ -88,8 +93,8 @@ class EggheadLessonIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
 
-        lesson = self._download_json(
-            'https://egghead.io/api/v1/lessons/%s' % display_id, display_id)
+        lesson = self._call_api(
+            'lessons/' + display_id, display_id, 'lesson')
 
         lesson_id = compat_str(lesson['id'])
         title = lesson['title']

From b111a64135244b73b86a1720e9a5212e726afcbf Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 2 Feb 2021 19:05:37 +0100
Subject: [PATCH 178/860] [egghead] fix typo

---
 youtube_dl/extractor/egghead.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/egghead.py b/youtube_dl/extractor/egghead.py
index 94dd75b9b..aff9b88c0 100644
--- a/youtube_dl/extractor/egghead.py
+++ b/youtube_dl/extractor/egghead.py
@@ -16,7 +16,7 @@ class EggheadBaseIE(InfoExtractor):
     def _call_api(self, path, video_id, resource, fatal=True):
         return self._download_json(
             'https://app.egghead.io/api/v1/' + path,
-            video_id, 'Downloading %s JSON' % resource)
+            video_id, 'Downloading %s JSON' % resource, fatal=fatal)
 
 
 class EggheadCourseIE(EggheadBaseIE):
@@ -79,7 +79,7 @@ class EggheadLessonIE(EggheadBaseIE):
             'upload_date': '20161209',
             'duration': 304,
             'view_count': 0,
-            'tags': ['free', 'javascript'],
+            'tags': 'count:2',
         },
         'params': {
             'skip_download': True,

From 1e2575df8714ce9056e559058a187ec0ffd2d739 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 3 Feb 2021 00:21:46 +0100
Subject: [PATCH 179/860] Credit @adrianheine for #27732

---
 AUTHORS | 1 +
 1 file changed, 1 insertion(+)

diff --git a/AUTHORS b/AUTHORS
index b507cb8df..4a6d7dacd 100644
--- a/AUTHORS
+++ b/AUTHORS
@@ -246,3 +246,4 @@ Enes Solak
 Nathan Rossi
 Thomas van der Berg
 Luca Cherubin
+Adrian Heine
\ No newline at end of file

From 07f7aad81c47a11483a357e53380fae1ffbadea9 Mon Sep 17 00:00:00 2001
From: Guillem Vela <guillemglez@gmail.com>
Date: Thu, 27 Feb 2020 22:18:47 +0100
Subject: [PATCH 180/860] [ccma] improve metadata extraction(closes #27994)

- extract age_limit, alt_title, categories, series and episode_number
- fix timestamp multiple subtitles extraction
---
 youtube_dl/extractor/ccma.py | 65 ++++++++++++++++++++++++++++++------
 1 file changed, 54 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/ccma.py b/youtube_dl/extractor/ccma.py
index 544647f92..4db51e650 100644
--- a/youtube_dl/extractor/ccma.py
+++ b/youtube_dl/extractor/ccma.py
@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import datetime
 import re
 
 from .common import InfoExtractor
@@ -8,8 +9,8 @@ from ..utils import (
     clean_html,
     int_or_none,
     parse_duration,
-    parse_iso8601,
     parse_resolution,
+    try_get,
     url_or_none,
 )
 
@@ -24,8 +25,9 @@ class CCMAIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'L\'espot de La Marató de TV3',
             'description': 'md5:f12987f320e2f6e988e9908e4fe97765',
-            'timestamp': 1470918540,
-            'upload_date': '20160811',
+            'timestamp': 1478608140,
+            'upload_date': '20161108',
+            'age_limit': 0,
         }
     }, {
         'url': 'http://www.ccma.cat/catradio/alacarta/programa/el-consell-de-savis-analitza-el-derbi/audio/943685/',
@@ -35,8 +37,24 @@ class CCMAIE(InfoExtractor):
             'ext': 'mp3',
             'title': 'El Consell de Savis analitza el derbi',
             'description': 'md5:e2a3648145f3241cb9c6b4b624033e53',
-            'upload_date': '20171205',
-            'timestamp': 1512507300,
+            'upload_date': '20170512',
+            'timestamp': 1494622500,
+            'vcodec': 'none',
+            'categories': ['Esports'],
+        }
+    }, {
+        'url': 'http://www.ccma.cat/tv3/alacarta/crims/crims-josep-tallada-lespereu-me-capitol-1/video/6031387/',
+        'md5': 'b43c3d3486f430f3032b5b160d80cbc3',
+        'info_dict': {
+            'id': '6031387',
+            'ext': 'mp4',
+            'title': 'Crims - Josep Talleda, l\'"Espereu-me" (capítol 1)',
+            'description': 'md5:7cbdafb640da9d0d2c0f62bad1e74e60',
+            'timestamp': 1582577700,
+            'upload_date': '20200224',
+            'subtitles': 'mincount:4',
+            'age_limit': 16,
+            'series': 'Crims',
         }
     }]
 
@@ -72,17 +90,27 @@ class CCMAIE(InfoExtractor):
 
         informacio = media['informacio']
         title = informacio['titol']
-        durada = informacio.get('durada', {})
+        durada = informacio.get('durada') or {}
         duration = int_or_none(durada.get('milisegons'), 1000) or parse_duration(durada.get('text'))
-        timestamp = parse_iso8601(informacio.get('data_emissio', {}).get('utc'))
+        tematica = try_get(informacio, lambda x: x['tematica']['text'])
+
+        timestamp = None
+        data_utc = try_get(informacio, lambda x: x['data_emissio']['utc'])
+        try:
+            timestamp = datetime.datetime.strptime(
+                data_utc, '%Y-%d-%mT%H:%M:%S%z').timestamp()
+        except TypeError:
+            pass
 
         subtitles = {}
-        subtitols = media.get('subtitols', {})
-        if subtitols:
-            sub_url = subtitols.get('url')
+        subtitols = media.get('subtitols') or []
+        if isinstance(subtitols, dict):
+            subtitols = [subtitols]
+        for st in subtitols:
+            sub_url = st.get('url')
             if sub_url:
                 subtitles.setdefault(
-                    subtitols.get('iso') or subtitols.get('text') or 'ca', []).append({
+                    st.get('iso') or st.get('text') or 'ca', []).append({
                         'url': sub_url,
                     })
 
@@ -97,6 +125,16 @@ class CCMAIE(InfoExtractor):
                     'height': int_or_none(imatges.get('alcada')),
                 }]
 
+        age_limit = None
+        codi_etic = try_get(informacio, lambda x: x['codi_etic']['id'])
+        if codi_etic:
+            codi_etic_s = codi_etic.split('_')
+            if len(codi_etic_s) == 2:
+                if codi_etic_s[1] == 'TP':
+                    age_limit = 0
+                else:
+                    age_limit = int_or_none(codi_etic_s[1])
+
         return {
             'id': media_id,
             'title': title,
@@ -106,4 +144,9 @@ class CCMAIE(InfoExtractor):
             'thumbnails': thumbnails,
             'subtitles': subtitles,
             'formats': formats,
+            'age_limit': age_limit,
+            'alt_title': informacio.get('titol_complet'),
+            'episode_number': int_or_none(informacio.get('capitol')),
+            'categories': [tematica] if tematica else None,
+            'series': informacio.get('programa'),
         }

From ab25f3f43196ca56964ba34ba4674fcb2d08f69a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 3 Feb 2021 17:15:31 +0100
Subject: [PATCH 181/860] [youtube] pass embed URL to get_video_info request

---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 75a007353..42b0f452c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1397,6 +1397,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'Refetching age-gated info webpage',
                     'unable to download video info webpage', query={
                         'video_id': video_id,
+                        'eurl': 'https://www.youtube.com/embed/' + video_id,
                     }, fatal=False)),
                 lambda x: x['player_response'][0],
                 compat_str) or '{}', video_id)

From 1b731ebcaa3ef2a1e52cf6968cf93e08d50fe0d4 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 3 Feb 2021 18:13:17 +0100
Subject: [PATCH 182/860] [bravotv] add support for oxygen.com(closes
 #13357)(closes #22500)

---
 youtube_dl/extractor/bravotv.py | 14 ++++++++++----
 1 file changed, 10 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/bravotv.py b/youtube_dl/extractor/bravotv.py
index b9715df00..bae2aedce 100644
--- a/youtube_dl/extractor/bravotv.py
+++ b/youtube_dl/extractor/bravotv.py
@@ -12,7 +12,7 @@ from ..utils import (
 
 
 class BravoTVIE(AdobePassIE):
-    _VALID_URL = r'https?://(?:www\.)?bravotv\.com/(?:[^/]+/)+(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<req_id>bravotv|oxygen)\.com/(?:[^/]+/)+(?P<id>[^/?#]+)'
     _TESTS = [{
         'url': 'https://www.bravotv.com/top-chef/season-16/episode-15/videos/the-top-chef-season-16-winner-is',
         'md5': 'e34684cfea2a96cd2ee1ef3a60909de9',
@@ -28,10 +28,13 @@ class BravoTVIE(AdobePassIE):
     }, {
         'url': 'http://www.bravotv.com/below-deck/season-3/ep-14-reunion-part-1',
         'only_matching': True,
+    }, {
+        'url': 'https://www.oxygen.com/in-ice-cold-blood/season-2/episode-16/videos/handling-the-horwitz-house-after-the-murder-season-2',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
+        site, display_id = re.match(self._VALID_URL, url).groups()
         webpage = self._download_webpage(url, display_id)
         settings = self._parse_json(self._search_regex(
             r'<script[^>]+data-drupal-selector="drupal-settings-json"[^>]*>({.+?})</script>', webpage, 'drupal settings'),
@@ -53,11 +56,14 @@ class BravoTVIE(AdobePassIE):
                 tp_path = release_pid = tve['release_pid']
             if tve.get('entitlement') == 'auth':
                 adobe_pass = settings.get('tve_adobe_auth', {})
+                if site == 'bravotv':
+                    site = 'bravo'
                 resource = self._get_mvpd_resource(
-                    adobe_pass.get('adobePassResourceId', 'bravo'),
+                    adobe_pass.get('adobePassResourceId') or site,
                     tve['title'], release_pid, tve.get('rating'))
                 query['auth'] = self._extract_mvpd_auth(
-                    url, release_pid, adobe_pass.get('adobePassRequestorId', 'bravo'), resource)
+                    url, release_pid,
+                    adobe_pass.get('adobePassRequestorId') or site, resource)
         else:
             shared_playlist = settings['ls_playlist']
             account_pid = shared_playlist['account_pid']

From 83031d749b11f062b9ba97023c228329e771cbd5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Feb 2021 00:25:26 +0700
Subject: [PATCH 183/860] [pornhub:user] Add support for URLs unavailable via
 /videos page and improve paging (closes #27853)

---
 youtube_dl/extractor/pornhub.py | 56 +++++++++++++++++++++++++++------
 1 file changed, 47 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 2fcbd186f..67e3731c8 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -22,6 +22,7 @@ from ..utils import (
     orderedSet,
     remove_quotes,
     str_to_int,
+    update_url_query,
     url_or_none,
 )
 
@@ -405,6 +406,10 @@ class PornHubIE(PornHubBaseIE):
 
 
 class PornHubPlaylistBaseIE(PornHubBaseIE):
+    def _extract_page(self, url):
+        return int_or_none(self._search_regex(
+            r'\bpage=(\d+)', url, 'page', default=None))
+
     def _extract_entries(self, webpage, host):
         # Only process container div with main playlist content skipping
         # drop-down menu that uses similar pattern for videos (see
@@ -463,14 +468,27 @@ class PornHubUserIE(PornHubPlaylistBaseIE):
     }, {
         'url': 'https://www.pornhub.com/model/zoe_ph?abc=1',
         'only_matching': True,
+    }, {
+        # Unavailable via /videos page, but available with direct pagination
+        # on pornstar page (see [1]), requires premium
+        # 1. https://github.com/ytdl-org/youtube-dl/issues/27853
+        'url': 'https://www.pornhubpremium.com/pornstar/sienna-west',
+        'only_matching': True,
+    }, {
+        # Same as before, multi page
+        'url': 'https://www.pornhubpremium.com/pornstar/lily-labeau',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         user_id = mobj.group('id')
+        videos_url = '%s/videos' % mobj.group('url')
+        page = self._extract_page(url)
+        if page:
+            videos_url = update_url_query(videos_url, {'page': page})
         return self.url_result(
-            '%s/videos' % mobj.group('url'), ie=PornHubPagedVideoListIE.ie_key(),
-            video_id=user_id)
+            videos_url, ie=PornHubPagedVideoListIE.ie_key(), video_id=user_id)
 
 
 class PornHubPagedPlaylistBaseIE(PornHubPlaylistBaseIE):
@@ -488,17 +506,37 @@ class PornHubPagedPlaylistBaseIE(PornHubPlaylistBaseIE):
         host = mobj.group('host')
         item_id = mobj.group('id')
 
-        page = int_or_none(self._search_regex(
-            r'\bpage=(\d+)', url, 'page', default=None))
+        page = self._extract_page(url)
+
+        VIDEOS = '/videos'
+
+        def download_page(base_url, num):
+            note = 'Downloading %spage %d' % ('' if VIDEOS in base_url else 'fallback ', num)
+            return self._download_webpage(
+                base_url, item_id, note, query={'page': num})
+
+        def is_404(e):
+            return isinstance(e.cause, compat_HTTPError) and e.cause.code == 404
 
         entries = []
-        for page_num in (page, ) if page is not None else itertools.count(1):
+        base_url = url
+        has_page = page is not None
+        first_page = page if has_page else 1
+        for page_num in (first_page, ) if has_page else itertools.count(first_page):
             try:
-                webpage = self._download_webpage(
-                    url, item_id, 'Downloading page %d' % page_num,
-                    query={'page': page_num})
+                try:
+                    webpage = download_page(base_url, page_num)
+                except ExtractorError as e:
+                    # Some sources may not be available via /videos page,
+                    # trying to fallback to main page pagination (see [1])
+                    # 1. https://github.com/ytdl-org/youtube-dl/issues/27853
+                    if is_404(e) and page_num == first_page and VIDEOS in base_url:
+                        base_url = base_url.replace(VIDEOS, '')
+                        webpage = download_page(base_url, page_num)
+                    else:
+                        raise
             except ExtractorError as e:
-                if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
+                if is_404(e) and page_num != first_page:
                     break
                 raise
             page_entries = self._extract_entries(webpage, host)

From e22ff4e35681a600ed61918beab8ed316728ec39 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Feb 2021 04:09:11 +0700
Subject: [PATCH 184/860] [pornhub] Add support for authentication (closes
 #18797, closes #21416, closes #24294)

---
 youtube_dl/extractor/pornhub.py | 106 +++++++++++++++++++++++---------
 1 file changed, 76 insertions(+), 30 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 67e3731c8..83307a233 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -23,6 +23,7 @@ from ..utils import (
     remove_quotes,
     str_to_int,
     update_url_query,
+    urlencode_postdata,
     url_or_none,
 )
 
@@ -53,6 +54,66 @@ class PornHubBaseIE(InfoExtractor):
 
         return webpage, urlh
 
+    def _real_initialize(self):
+        self._logged_in = False
+
+    def _login(self, host):
+        if self._logged_in:
+            return
+
+        site = host.split('.')[0]
+
+        # Both sites pornhub and pornhubpremium have separate accounts
+        # so there should be an option to provide credentials for both.
+        # At the same time some videos are available under the same video id
+        # on both sites so that we have to identify them as the same video.
+        # For that purpose we have to keep both in the same extractor
+        # but under different netrc machines.
+        username, password = self._get_login_info(netrc_machine=site)
+        if username is None:
+            return
+
+        login_url = 'https://www.%s/%slogin' % (host, 'premium/' if 'premium' in host else '')
+        login_page = self._download_webpage(
+            login_url, None, 'Downloading %s login page' % site)
+
+        def is_logged(webpage):
+            return any(re.search(p, webpage) for p in (
+                r'class=["\']signOut',
+                r'>Sign\s+[Oo]ut\s*<'))
+
+        if is_logged(login_page):
+            self._logged_in = True
+            return
+
+        login_form = self._hidden_inputs(login_page)
+
+        login_form.update({
+            'username': username,
+            'password': password,
+        })
+
+        response = self._download_json(
+            'https://www.%s/front/authenticate' % host, None,
+            'Logging in to %s' % site,
+            data=urlencode_postdata(login_form),
+            headers={
+                'Content-Type': 'application/x-www-form-urlencoded; charset=UTF-8',
+                'Referer': login_url,
+                'X-Requested-With': 'XMLHttpRequest',
+            })
+
+        if response.get('success') == '1':
+            self._logged_in = True
+            return
+
+        message = response.get('message')
+        if message is not None:
+            raise ExtractorError(
+                'Unable to login: %s' % message, expected=True)
+
+        raise ExtractorError('Unable to log in')
+
 
 class PornHubIE(PornHubBaseIE):
     IE_DESC = 'PornHub and Thumbzilla'
@@ -164,12 +225,20 @@ class PornHubIE(PornHubBaseIE):
     }, {
         'url': 'https://www.pornhubpremium.com/view_video.php?viewkey=ph5e4acdae54a82',
         'only_matching': True,
+    }, {
+        # Some videos are available with the same id on both premium
+        # and non-premium sites (e.g. this and the following test)
+        'url': 'https://www.pornhub.com/view_video.php?viewkey=ph5f75b0f4b18e3',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.pornhubpremium.com/view_video.php?viewkey=ph5f75b0f4b18e3',
+        'only_matching': True,
     }]
 
     @staticmethod
     def _extract_urls(webpage):
         return re.findall(
-            r'<iframe[^>]+?src=["\'](?P<url>(?:https?:)?//(?:www\.)?pornhub\.(?:com|net|org)/embed/[\da-z]+)',
+            r'<iframe[^>]+?src=["\'](?P<url>(?:https?:)?//(?:www\.)?pornhub(?:premium)?\.(?:com|net|org)/embed/[\da-z]+)',
             webpage)
 
     def _extract_count(self, pattern, webpage, name):
@@ -181,12 +250,7 @@ class PornHubIE(PornHubBaseIE):
         host = mobj.group('host') or 'pornhub.com'
         video_id = mobj.group('id')
 
-        if 'premium' in host:
-            if not self._downloader.params.get('cookiefile'):
-                raise ExtractorError(
-                    'PornHub Premium requires authentication.'
-                    ' You may want to use --cookies.',
-                    expected=True)
+        self._login(host)
 
         self._set_cookie(host, 'age_verified', '1')
 
@@ -427,26 +491,6 @@ class PornHubPlaylistBaseIE(PornHubBaseIE):
                 container))
         ]
 
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        host = mobj.group('host')
-        playlist_id = mobj.group('id')
-
-        webpage = self._download_webpage(url, playlist_id)
-
-        entries = self._extract_entries(webpage, host)
-
-        playlist = self._parse_json(
-            self._search_regex(
-                r'(?:playlistObject|PLAYLIST_VIEW)\s*=\s*({.+?});', webpage,
-                'playlist', default='{}'),
-            playlist_id, fatal=False)
-        title = playlist.get('title') or self._search_regex(
-            r'>Videos\s+in\s+(.+?)\s+[Pp]laylist<', webpage, 'title', fatal=False)
-
-        return self.playlist_result(
-            entries, playlist_id, title, playlist.get('description'))
-
 
 class PornHubUserIE(PornHubPlaylistBaseIE):
     _VALID_URL = r'(?P<url>https?://(?:[^/]+\.)?(?P<host>pornhub(?:premium)?\.(?:com|net|org))/(?:(?:user|channel)s|model|pornstar)/(?P<id>[^/?#&]+))(?:[?#&]|/(?!videos)|$)'
@@ -506,12 +550,14 @@ class PornHubPagedPlaylistBaseIE(PornHubPlaylistBaseIE):
         host = mobj.group('host')
         item_id = mobj.group('id')
 
+        self._login(host)
+
         page = self._extract_page(url)
 
         VIDEOS = '/videos'
 
-        def download_page(base_url, num):
-            note = 'Downloading %spage %d' % ('' if VIDEOS in base_url else 'fallback ', num)
+        def download_page(base_url, num, fallback=False):
+            note = 'Downloading page %d%s' % (num, ' (switch to fallback)' if fallback else '')
             return self._download_webpage(
                 base_url, item_id, note, query={'page': num})
 
@@ -532,7 +578,7 @@ class PornHubPagedPlaylistBaseIE(PornHubPlaylistBaseIE):
                     # 1. https://github.com/ytdl-org/youtube-dl/issues/27853
                     if is_404(e) and page_num == first_page and VIDEOS in base_url:
                         base_url = base_url.replace(VIDEOS, '')
-                        webpage = download_page(base_url, page_num)
+                        webpage = download_page(base_url, page_num, fallback=True)
                     else:
                         raise
             except ExtractorError as e:

From 1f0910bc2742b16be8425841d5ed6a0fd96f82a4 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Feb 2021 04:17:45 +0700
Subject: [PATCH 185/860] [svtplay] Fix video id extraction (closes #28058)

---
 youtube_dl/extractor/svt.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index a0b6ef4db..4acc29fce 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -255,8 +255,10 @@ class SVTPlayIE(SVTPlayBaseIE):
             svt_id = self._search_regex(
                 (r'<video[^>]+data-video-id=["\']([\da-zA-Z-]+)',
                  r'["\']videoSvtId["\']\s*:\s*["\']([\da-zA-Z-]+)',
+                 r'["\']videoSvtId\\?["\']\s*:\s*\\?["\']([\da-zA-Z-]+)',
                  r'"content"\s*:\s*{.*?"id"\s*:\s*"([\da-zA-Z-]+)"',
-                 r'["\']svtId["\']\s*:\s*["\']([\da-zA-Z-]+)'),
+                 r'["\']svtId["\']\s*:\s*["\']([\da-zA-Z-]+)',
+                 r'["\']svtId\\?["\']\s*:\s*\\?["\']([\da-zA-Z-]+)'),
                 webpage, 'video id')
 
         info_dict = self._extract_by_video_id(svt_id, webpage)

From 2adc0c51cdf38e039fba0ede11f65bbd9c71bde8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Feb 2021 04:20:09 +0700
Subject: [PATCH 186/860] [pornhub] Add placeholder netrc machine

---
 youtube_dl/extractor/pornhub.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 83307a233..83773aebb 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -29,6 +29,8 @@ from ..utils import (
 
 
 class PornHubBaseIE(InfoExtractor):
+    _NETRC_MACHINE = 'pornhub'
+
     def _download_webpage_handle(self, *args, **kwargs):
         def dl(*args, **kwargs):
             return super(PornHubBaseIE, self)._download_webpage_handle(*args, **kwargs)

From 89c5a7d5aabd138a14c76453d79d5d66ef573bde Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Feb 2021 04:36:57 +0700
Subject: [PATCH 187/860] [pornhub] Implement lazy playlist extraction

---
 youtube_dl/extractor/pornhub.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 83773aebb..b7631e4e1 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -547,13 +547,7 @@ class PornHubPagedPlaylistBaseIE(PornHubPlaylistBaseIE):
                 <button[^>]+\bid=["\']moreDataBtn
             ''', webpage) is not None
 
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        host = mobj.group('host')
-        item_id = mobj.group('id')
-
-        self._login(host)
-
+    def _entries(self, url, host, item_id):
         page = self._extract_page(url)
 
         VIDEOS = '/videos'
@@ -566,7 +560,6 @@ class PornHubPagedPlaylistBaseIE(PornHubPlaylistBaseIE):
         def is_404(e):
             return isinstance(e.cause, compat_HTTPError) and e.cause.code == 404
 
-        entries = []
         base_url = url
         has_page = page is not None
         first_page = page if has_page else 1
@@ -590,11 +583,19 @@ class PornHubPagedPlaylistBaseIE(PornHubPlaylistBaseIE):
             page_entries = self._extract_entries(webpage, host)
             if not page_entries:
                 break
-            entries.extend(page_entries)
+            for e in page_entries:
+                yield e
             if not self._has_more(webpage):
                 break
 
-        return self.playlist_result(orderedSet(entries), item_id)
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        host = mobj.group('host')
+        item_id = mobj.group('id')
+
+        self._login(host)
+
+        return self.playlist_result(self._entries(url, host, item_id), item_id)
 
 
 class PornHubPagedVideoListIE(PornHubPagedPlaylistBaseIE):

From 3c07d007ca5376719a0cfe6b9c6627b38cbd3e1f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Feb 2021 04:47:30 +0700
Subject: [PATCH 188/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 42 ++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 42 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 7f2e0aad1..bd753d524 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,45 @@
+version <unreleased>
+
+Extractors
+* [pornhub] Implement lazy playlist extraction
+* [svtplay] Fix video id extraction (#28058)
++ [pornhub] Add support for authentication (#18797, #21416, #24294)
+* [pornhub:user] Improve paging
++ [pornhub:user] Add support for URLs unavailable via /videos page (#27853)
++ [bravotv] Add support for oxygen.com (#13357, #22500)
++ [youtube] Pass embed URL to get_video_info request
+* [ccma] Improve metadata extraction (#27994)
+    + Extract age limit, alt title, categories, series and episode number
+    * Fix timestamp multiple subtitles extraction
+* [egghead] Update API domain (#28038)
+- [vidzi] Remove extractor (#12629)
+* [vidio] Improve metadata extraction
+* [youtube] Improve subtitles extraction
+* [youtube] Fix chapter extraction fallback
+* [youtube] Rewrite extractor
+    * Improve format sorting
+    * Remove unused code
+    * Fix series metadata extraction
+    * Fix trailer video extraction
+    * Improve error reporting
+    + Extract video location
++ [vvvvid] Add support for youtube embeds (#27825)
+* [googledrive] Report download page errors (#28005)
+* [vlive] Fix error message decoding for python 2 (#28004)
+* [youtube] Improve DASH formats file size extraction
+* [cda] Improve birth validation detection (#14022, #27929)
++ [awaan] Extract uploader id (#27963)
++ [medialaan] Add support DPG Media MyChannels based websites (#14871, #15597,
+  #16106, #16489)
+* [abcnews] Fix extraction (#12394, #27920)
+* [AMP] Fix upload date and timestamp extraction (#27970)
+* [tv4] Relax URL regular expression (#27964)
++ [tv2] Add support for mtvuutiset.fi (#27744)
+* [adn] Improve login warning reporting
+* [zype] Fix uplynk id extraction (#27956)
++ [adn] Add support for authentication (#17091, #27841, #27937)
+
+
 version 2021.01.24.1
 
 Core

From cfefb7d854f87e02c971170fcfa08f3ff2cb1bfa Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Feb 2021 04:49:25 +0700
Subject: [PATCH 189/860] release 2021.02.04

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 docs/supportedsites.md                           | 3 ++-
 youtube_dl/version.py                            | 2 +-
 8 files changed, 16 insertions(+), 15 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 2dde97a2c..86e48bc4e 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.01.24.1**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.04**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.01.24.1
+ [debug] youtube-dl version 2021.02.04
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index c520d1ee0..fa369b744 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.01.24.1**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.04**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 4aacd3bdc..806c7c58d 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.01.24.1**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.04**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index 91bbed506..1d1a36dda 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.01.24.1**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.04**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.01.24.1
+ [debug] youtube-dl version 2021.02.04
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index a0a2c989a..c19052a7a 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.01.24.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.01.24.1**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.04**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index bd753d524..d5d9c00a2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.02.04
 
 Extractors
 * [pornhub] Implement lazy playlist extraction
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 13bac6e27..e1b85b1d1 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -537,6 +537,7 @@
  - **mtv:video**
  - **mtvjapan**
  - **mtvservices:embedded**
+ - **MTVUutisetArticle**
  - **MuenchenTV**: münchen.tv
  - **mva**: Microsoft Virtual Academy videos
  - **mva:course**: Microsoft Virtual Academy courses
@@ -1058,7 +1059,6 @@
  - **vidme**
  - **vidme:user**
  - **vidme:user:likes**
- - **Vidzi**
  - **vier**: vier.be and vijf.be
  - **vier:videos**
  - **viewlift**
@@ -1103,6 +1103,7 @@
  - **vrv**
  - **vrv:series**
  - **VShare**
+ - **VTM**
  - **VTXTV**
  - **vube**: Vube.com
  - **VuClip**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index c52f1d9ca..d898525c9 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.01.24.1'
+__version__ = '2021.02.04'

From fc88e8f0e3e66f17f787cbc1ea45c87fdc70781e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adrian=20Heine=20n=C3=A9=20Lang?= <mail@adrianheine.de>
Date: Thu, 4 Feb 2021 00:57:56 +0100
Subject: [PATCH 190/860] [azmedien] Fix extraction (#28064)

---
 youtube_dl/extractor/azmedien.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/azmedien.py b/youtube_dl/extractor/azmedien.py
index b1e20def5..930266990 100644
--- a/youtube_dl/extractor/azmedien.py
+++ b/youtube_dl/extractor/azmedien.py
@@ -47,7 +47,7 @@ class AZMedienIE(InfoExtractor):
         'url': 'https://www.telebaern.tv/telebaern-news/montag-1-oktober-2018-ganze-sendung-133531189#video=0_7xjo9lf1',
         'only_matching': True
     }]
-    _API_TEMPL = 'https://www.%s/api/pub/gql/%s/NewsArticleTeaser/cb9f2f81ed22e9b47f4ca64ea3cc5a5d13e88d1d'
+    _API_TEMPL = 'https://www.%s/api/pub/gql/%s/NewsArticleTeaser/a4016f65fe62b81dc6664dd9f4910e4ab40383be'
     _PARTNER_ID = '1719221'
 
     def _real_extract(self, url):

From 7215691ab7cabc858b17c16928c372da3e35ec59 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Feb 2021 13:07:43 +0700
Subject: [PATCH 191/860] [youtube] Prefer DASH formats (closes #28070)

---
 youtube_dl/extractor/youtube.py | 12 +++++++++---
 1 file changed, 9 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 42b0f452c..a3b10c094 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1549,16 +1549,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if self._downloader.params.get('youtube_include_dash_manifest'):
             dash_manifest_url = streaming_data.get('dashManifestUrl')
             if dash_manifest_url:
+                dash_formats = []
                 for f in self._extract_mpd_formats(
                         dash_manifest_url, video_id, fatal=False):
-                    if f['format_id'] in itags:
-                        continue
                     filesize = int_or_none(self._search_regex(
                         r'/clen/(\d+)', f.get('fragment_base_url')
                         or f['url'], 'file size', default=None))
                     if filesize:
                         f['filesize'] = filesize
-                    formats.append(f)
+                    dash_formats.append(f)
+                # Until further investigation prefer DASH formats as non-DASH
+                # may not be available (see [1])
+                # 1. https://github.com/ytdl-org/youtube-dl/issues/28070
+                if dash_formats:
+                    dash_formats_keys = [f['format_id'] for f in dash_formats]
+                    formats = [f for f in formats if f['format_id'] not in dash_formats_keys]
+                    formats.extend(dash_formats)
 
         if not formats:
             if streaming_data.get('licenseInfos'):

From c7d407bca205d8eb248b94b611435187265b79da Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Feb 2021 13:09:28 +0700
Subject: [PATCH 192/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index d5d9c00a2..4392a4e6f 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,10 @@
+version <unreleased>
+
+Extractors
+* [youtube] Prefer DASH formats (#28070)
+* [azmedien] Fix extraction (#28064)
+
+
 version 2021.02.04
 
 Extractors

From a4bdc3112bf0e925afc2e512d5f23f9097f6bc7a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 4 Feb 2021 13:11:33 +0700
Subject: [PATCH 193/860] release 2021.02.04.1

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 youtube_dl/version.py                            | 2 +-
 7 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 86e48bc4e..19b750f86 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.02.04**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.04.1**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.02.04
+ [debug] youtube-dl version 2021.02.04.1
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index fa369b744..8acb80b60 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.02.04**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.04.1**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 806c7c58d..66edcf752 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.02.04**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.04.1**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index 1d1a36dda..18203fb34 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.02.04**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.04.1**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.02.04
+ [debug] youtube-dl version 2021.02.04.1
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index c19052a7a..20df40cc5 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.02.04**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.04.1**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index 4392a4e6f..784b73d8d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.02.04.1
 
 Extractors
 * [youtube] Prefer DASH formats (#28070)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index d898525c9..425f15589 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.02.04'
+__version__ = '2021.02.04.1'

From 1641b132323b544b9ae0dad06707425eba1f926b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 4 Feb 2021 13:05:35 +0100
Subject: [PATCH 194/860] [youtube] skip OTF formats(#28070)

---
 youtube_dl/extractor/youtube.py | 28 +++++++++++++++++-----------
 1 file changed, 17 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a3b10c094..eb5f70763 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1477,6 +1477,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         formats = []
         itags = []
+        itag_qualities = {}
         player_url = None
         q = qualities(['tiny', 'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'])
         streaming_data = player_response.get('streamingData') or {}
@@ -1486,6 +1487,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if fmt.get('targetDurationSec') or fmt.get('drmFamilies'):
                 continue
 
+            itag = str_or_none(fmt.get('itag'))
+            quality = fmt.get('quality')
+            if itag and quality:
+                itag_qualities[itag] = quality
+            # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
+            # (adding `&sq=0` to the URL) and parsing emsg box to determine the
+            # number of fragment that would subsequently requested with (`&sq=N`)
+            if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
+                continue
+
             fmt_url = fmt.get('url')
             if not fmt_url:
                 sc = compat_parse_qs(fmt.get('signatureCipher'))
@@ -1505,10 +1516,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 sp = try_get(sc, lambda x: x['sp'][0]) or 'signature'
                 fmt_url += '&' + sp + '=' + signature
 
-            itag = str_or_none(fmt.get('itag'))
             if itag:
                 itags.append(itag)
-            quality = fmt.get('quality')
             dct = {
                 'asr': int_or_none(fmt.get('audioSampleRate')),
                 'filesize': int_or_none(fmt.get('contentLength')),
@@ -1549,22 +1558,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if self._downloader.params.get('youtube_include_dash_manifest'):
             dash_manifest_url = streaming_data.get('dashManifestUrl')
             if dash_manifest_url:
-                dash_formats = []
                 for f in self._extract_mpd_formats(
                         dash_manifest_url, video_id, fatal=False):
+                    itag = f['format_id']
+                    if itag in itags:
+                        continue
+                    if itag in itag_qualities:
+                        f['quality'] = q(itag_qualities[itag])
                     filesize = int_or_none(self._search_regex(
                         r'/clen/(\d+)', f.get('fragment_base_url')
                         or f['url'], 'file size', default=None))
                     if filesize:
                         f['filesize'] = filesize
-                    dash_formats.append(f)
-                # Until further investigation prefer DASH formats as non-DASH
-                # may not be available (see [1])
-                # 1. https://github.com/ytdl-org/youtube-dl/issues/28070
-                if dash_formats:
-                    dash_formats_keys = [f['format_id'] for f in dash_formats]
-                    formats = [f for f in formats if f['format_id'] not in dash_formats_keys]
-                    formats.extend(dash_formats)
+                    formats.append(f)
 
         if not formats:
             if streaming_data.get('licenseInfos'):

From 0156ce95c5ba83de6c68a149d352ccecd983a294 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2021 01:54:46 +0700
Subject: [PATCH 195/860] [youtube] Extract abr and vbr (closes #28100)

---
 youtube_dl/extractor/youtube.py | 14 +++++++++++---
 1 file changed, 11 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index eb5f70763..b5e0f4eaa 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -500,6 +500,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'AfrojackVEVO',
                 'uploader_id': 'AfrojackVEVO',
                 'upload_date': '20131011',
+                'abr': 129.495,
             },
             'params': {
                 'youtube_include_dash_manifest': True,
@@ -1518,6 +1519,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
             if itag:
                 itags.append(itag)
+            tbr = float_or_none(
+                fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
             dct = {
                 'asr': int_or_none(fmt.get('audioSampleRate')),
                 'filesize': int_or_none(fmt.get('contentLength')),
@@ -1526,8 +1529,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'fps': int_or_none(fmt.get('fps')),
                 'height': int_or_none(fmt.get('height')),
                 'quality': q(quality),
-                'tbr': float_or_none(fmt.get(
-                    'averageBitrate') or fmt.get('bitrate'), 1000),
+                'tbr': tbr,
                 'url': fmt_url,
                 'width': fmt.get('width'),
             }
@@ -1538,7 +1540,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 if mobj:
                     dct['ext'] = mimetype2ext(mobj.group(1))
                     dct.update(parse_codecs(mobj.group(2)))
-            if dct.get('acodec') == 'none' or dct.get('vcodec') == 'none':
+            no_audio = dct.get('acodec') == 'none'
+            no_video = dct.get('vcodec') == 'none'
+            if no_audio:
+                dct['vbr'] = tbr
+            if no_video:
+                dct['abr'] = tbr
+            if no_audio or no_video:
                 dct['downloader_options'] = {
                     # Youtube throttles chunks >~10M
                     'http_chunk_size': 10485760,

From 0cf09c2b4168cb99800836d8c1ff0d6d8b16fb6e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2021 02:17:03 +0700
Subject: [PATCH 196/860] [youtube] Fix release date extraction (closes #28094)

---
 youtube_dl/extractor/youtube.py | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b5e0f4eaa..c87e54e6b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1019,6 +1019,28 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'url': 'https://www.youtube.com/watch_popup?v=63RmMXCd_bQ',
             'only_matching': True,
         },
+        {
+            # https://github.com/ytdl-org/youtube-dl/pull/28094
+            'url': 'OtqTfy26tG0',
+            'info_dict': {
+                'id': 'OtqTfy26tG0',
+                'ext': 'mp4',
+                'title': 'Burn Out',
+                'description': 'md5:8d07b84dcbcbfb34bc12a56d968b6131',
+                'upload_date': '20141120',
+                'uploader': 'The Cinematic Orchestra - Topic',
+                'uploader_id': 'UCIzsJBIyo8hhpFm1NK0uLgw',
+                'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCIzsJBIyo8hhpFm1NK0uLgw',
+                'artist': 'The Cinematic Orchestra',
+                'track': 'Burn Out',
+                'album': 'Every Day',
+                'release_data': None,
+                'release_year': None,
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
     ]
     _formats = {
         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
@@ -1743,7 +1765,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'artist': mobj.group('clean_artist') or ', '.join(a.strip() for a in mobj.group('artist').split('·')),
                     'track': mobj.group('track').strip(),
                     'release_date': release_date,
-                    'release_year': int(release_year),
+                    'release_year': int_or_none(release_year),
                 })
 
         initial_data = None

From 240585470539d31d9c3785a67861491fa3696451 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Adrian=20Heine=20n=C3=A9=20Lang?= <mail@adrianheine.de>
Date: Sat, 6 Feb 2021 20:46:05 +0100
Subject: [PATCH 197/860] [urplay] Fix extraction (closes #28073) (#28074)

---
 youtube_dl/extractor/urplay.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/urplay.py b/youtube_dl/extractor/urplay.py
index 10b817760..5452c7ca1 100644
--- a/youtube_dl/extractor/urplay.py
+++ b/youtube_dl/extractor/urplay.py
@@ -42,8 +42,8 @@ class URPlayIE(InfoExtractor):
         url = url.replace('skola.se/Produkter', 'play.se/program')
         webpage = self._download_webpage(url, video_id)
         urplayer_data = self._parse_json(self._html_search_regex(
-            r'data-react-class="components/Player/Player"[^>]+data-react-props="({.+?})"',
-            webpage, 'urplayer data'), video_id)['currentProduct']
+            r'data-react-class="routes/Product/components/ProgramContainer/ProgramContainer"[^>]+data-react-props="({.+?})"',
+            webpage, 'urplayer data'), video_id)['accessibleEpisodes'][0]
         episode = urplayer_data['title']
         raw_streaming_info = urplayer_data['streamingInfo']['raw']
         host = self._download_json(

From 7a9161578e42abe681c9d3352ecc9a18a9b8df6a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2021 19:18:06 +0700
Subject: [PATCH 198/860] [cda] Detect geo restricted videos (refs #28106)

---
 youtube_dl/extractor/cda.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/cda.py b/youtube_dl/extractor/cda.py
index 6429454fb..1b4362144 100644
--- a/youtube_dl/extractor/cda.py
+++ b/youtube_dl/extractor/cda.py
@@ -95,6 +95,9 @@ class CDAIE(InfoExtractor):
         if 'Ten film jest dostępny dla użytkowników premium' in webpage:
             raise ExtractorError('This video is only available for premium users.', expected=True)
 
+        if re.search(r'niedostępn[ey] w(?:&nbsp;|\s+)Twoim kraju\s*<', webpage):
+            self.raise_geo_restricted()
+
         need_confirm_age = False
         if self._html_search_regex(r'(<form[^>]+action="[^"]*/a/validatebirth[^"]*")',
                                    webpage, 'birthday validate form', default=None):

From 5fc53690cbe6abb11941a3f4846b566a7472753e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 7 Feb 2021 20:34:41 +0700
Subject: [PATCH 199/860] [archiveorg] Fix and improve extraction (closes
 #21330, closes #23586, closes #25277, closes #26780, closes #27109, closes
 #27236, closes #28063)

---
 youtube_dl/extractor/archiveorg.py | 54 +++++++++++++++++++++++-------
 1 file changed, 42 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/archiveorg.py b/youtube_dl/extractor/archiveorg.py
index c79c58e82..e42ed5e79 100644
--- a/youtube_dl/extractor/archiveorg.py
+++ b/youtube_dl/extractor/archiveorg.py
@@ -2,15 +2,17 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    unified_strdate,
     clean_html,
+    extract_attributes,
+    unified_strdate,
+    unified_timestamp,
 )
 
 
 class ArchiveOrgIE(InfoExtractor):
     IE_NAME = 'archive.org'
     IE_DESC = 'archive.org videos'
-    _VALID_URL = r'https?://(?:www\.)?archive\.org/(?:details|embed)/(?P<id>[^/?#]+)(?:[?].*)?$'
+    _VALID_URL = r'https?://(?:www\.)?archive\.org/(?:details|embed)/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'http://archive.org/details/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
         'md5': '8af1d4cf447933ed3c7f4871162602db',
@@ -19,8 +21,11 @@ class ArchiveOrgIE(InfoExtractor):
             'ext': 'ogg',
             'title': '1968 Demo - FJCC Conference Presentation Reel #1',
             'description': 'md5:da45c349df039f1cc8075268eb1b5c25',
-            'upload_date': '19681210',
-            'uploader': 'SRI International'
+            'creator': 'SRI International',
+            'release_date': '19681210',
+            'uploader': 'SRI International',
+            'timestamp': 1268695290,
+            'upload_date': '20100315',
         }
     }, {
         'url': 'https://archive.org/details/Cops1922',
@@ -29,22 +34,43 @@ class ArchiveOrgIE(InfoExtractor):
             'id': 'Cops1922',
             'ext': 'mp4',
             'title': 'Buster Keaton\'s "Cops" (1922)',
-            'description': 'md5:89e7c77bf5d965dd5c0372cfb49470f6',
+            'description': 'md5:43a603fd6c5b4b90d12a96b921212b9c',
+            'timestamp': 1387699629,
+            'upload_date': '20131222',
         }
     }, {
         'url': 'http://archive.org/embed/XD300-23_68HighlightsAResearchCntAugHumanIntellect',
         'only_matching': True,
+    }, {
+        'url': 'https://archive.org/details/MSNBCW_20131125_040000_To_Catch_a_Predator/',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(
             'http://archive.org/embed/' + video_id, video_id)
-        jwplayer_playlist = self._parse_json(self._search_regex(
-            r"(?s)Play\('[^']+'\s*,\s*(\[.+\])\s*,\s*{.*?}\)",
-            webpage, 'jwplayer playlist'), video_id)
-        info = self._parse_jwplayer_data(
-            {'playlist': jwplayer_playlist}, video_id, base_url=url)
+
+        playlist = None
+        play8 = self._search_regex(
+            r'(<[^>]+\bclass=["\']js-play8-playlist[^>]+>)', webpage,
+            'playlist', default=None)
+        if play8:
+            attrs = extract_attributes(play8)
+            playlist = attrs.get('value')
+        if not playlist:
+            # Old jwplayer fallback
+            playlist = self._search_regex(
+                r"(?s)Play\('[^']+'\s*,\s*(\[.+\])\s*,\s*{.*?}\)",
+                webpage, 'jwplayer playlist', default='[]')
+        jwplayer_playlist = self._parse_json(playlist, video_id, fatal=False)
+        if jwplayer_playlist:
+            info = self._parse_jwplayer_data(
+                {'playlist': jwplayer_playlist}, video_id, base_url=url)
+        else:
+            # HTML5 media fallback
+            info = self._parse_html5_media_entries(url, webpage, video_id)[0]
+            info['id'] = video_id
 
         def get_optional(metadata, field):
             return metadata.get(field, [None])[0]
@@ -58,8 +84,12 @@ class ArchiveOrgIE(InfoExtractor):
             'description': clean_html(get_optional(metadata, 'description')),
         })
         if info.get('_type') != 'playlist':
+            creator = get_optional(metadata, 'creator')
             info.update({
-                'uploader': get_optional(metadata, 'creator'),
-                'upload_date': unified_strdate(get_optional(metadata, 'date')),
+                'creator': creator,
+                'release_date': unified_strdate(get_optional(metadata, 'date')),
+                'uploader': get_optional(metadata, 'publisher') or creator,
+                'timestamp': unified_timestamp(get_optional(metadata, 'publicdate')),
+                'language': get_optional(metadata, 'language'),
             })
         return info

From 99c68db0a8adc634e2e928ea2756a2ceee3ae863 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 8 Feb 2021 09:20:28 +0100
Subject: [PATCH 200/860] [youtube] add support phone/tablet JS player(closes
 #26424)

---
 test/test_youtube_signature.py  | 31 ++++++++++---------------------
 youtube_dl/extractor/youtube.py |  6 +++++-
 2 files changed, 15 insertions(+), 22 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index b5a4d0d5f..627d4cb92 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -19,55 +19,46 @@ from youtube_dl.compat import compat_str, compat_urlretrieve
 _TESTS = [
     (
         'https://s.ytimg.com/yts/jsbin/html5player-vflHOr_nV.js',
-        'js',
         86,
         '>=<;:/.-[+*)(\'&%$#"!ZYX0VUTSRQPONMLKJIHGFEDCBA\\yxwvutsrqponmlkjihgfedcba987654321',
     ),
     (
         'https://s.ytimg.com/yts/jsbin/html5player-vfldJ8xgI.js',
-        'js',
         85,
         '3456789a0cdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRS[UVWXYZ!"#$%&\'()*+,-./:;<=>?@',
     ),
     (
         'https://s.ytimg.com/yts/jsbin/html5player-vfle-mVwz.js',
-        'js',
         90,
         ']\\[@?>=<;:/.-,+*)(\'&%$#"hZYXWVUTSRQPONMLKJIHGFEDCBAzyxwvutsrqponmlkjiagfedcb39876',
     ),
     (
         'https://s.ytimg.com/yts/jsbin/html5player-en_US-vfl0Cbn9e.js',
-        'js',
         84,
         'O1I3456789abcde0ghijklmnopqrstuvwxyzABCDEFGHfJKLMN2PQRSTUVW@YZ!"#$%&\'()*+,-./:;<=',
     ),
     (
         'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflXGBaUN.js',
-        'js',
         '2ACFC7A61CA478CD21425E5A57EBD73DDC78E22A.2094302436B2D377D14A3BBA23022D023B8BC25AA',
         'A52CB8B320D22032ABB3A41D773D2B6342034902.A22E87CDD37DBE75A5E52412DC874AC16A7CFCA2',
     ),
     (
         'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflBb0OQx.js',
-        'js',
         84,
         '123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQ0STUVWXYZ!"#$%&\'()*+,@./:;<=>'
     ),
     (
         'https://s.ytimg.com/yts/jsbin/html5player-en_US-vfl9FYC6l.js',
-        'js',
         83,
         '123456789abcdefghijklmnopqr0tuvwxyzABCDETGHIJKLMNOPQRS>UVWXYZ!"#$%&\'()*+,-./:;<=F'
     ),
     (
         'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflCGk6yw/html5player.js',
-        'js',
         '4646B5181C6C3020DF1D9C7FCFEA.AD80ABF70C39BD369CCCAE780AFBB98FA6B6CB42766249D9488C288',
         '82C8849D94266724DC6B6AF89BBFA087EACCD963.B93C07FBA084ACAEFCF7C9D1FD0203C6C1815B6B'
     ),
     (
         'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js',
-        'js',
         '312AA52209E3623129A412D56A40F11CB0AF14AE.3EE09501CB14E3BCDC3B2AE808BF3F1D14E7FBF12',
         '112AA5220913623229A412D56A40F11CB0AF14AE.3EE0950FCB14EEBCDC3B2AE808BF331D14E7FBF3',
     )
@@ -78,6 +69,10 @@ class TestPlayerInfo(unittest.TestCase):
     def test_youtube_extract_player_info(self):
         PLAYER_URLS = (
             ('https://www.youtube.com/s/player/64dddad9/player_ias.vflset/en_US/base.js', '64dddad9'),
+            ('https://www.youtube.com/s/player/64dddad9/player_ias.vflset/fr_FR/base.js', '64dddad9'),
+            ('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-phone-en_US.vflset/base.js', '64dddad9'),
+            ('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-phone-de_DE.vflset/base.js', '64dddad9'),
+            ('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-tablet-en_US.vflset/base.js', '64dddad9'),
             # obsolete
             ('https://www.youtube.com/yts/jsbin/player_ias-vfle4-e03/en_US/base.js', 'vfle4-e03'),
             ('https://www.youtube.com/yts/jsbin/player_ias-vfl49f_g4/en_US/base.js', 'vfl49f_g4'),
@@ -100,13 +95,13 @@ class TestSignature(unittest.TestCase):
             os.mkdir(self.TESTDATA_DIR)
 
 
-def make_tfunc(url, stype, sig_input, expected_sig):
+def make_tfunc(url, sig_input, expected_sig):
     m = re.match(r'.*-([a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$', url)
     assert m, '%r should follow URL format' % url
     test_id = m.group(1)
 
     def test_func(self):
-        basename = 'player-%s.%s' % (test_id, stype)
+        basename = 'player-%s.js' % test_id
         fn = os.path.join(self.TESTDATA_DIR, basename)
 
         if not os.path.exists(fn):
@@ -114,22 +109,16 @@ def make_tfunc(url, stype, sig_input, expected_sig):
 
         ydl = FakeYDL()
         ie = YoutubeIE(ydl)
-        if stype == 'js':
-            with io.open(fn, encoding='utf-8') as testf:
-                jscode = testf.read()
-            func = ie._parse_sig_js(jscode)
-        else:
-            assert stype == 'swf'
-            with open(fn, 'rb') as testf:
-                swfcode = testf.read()
-            func = ie._parse_sig_swf(swfcode)
+        with io.open(fn, encoding='utf-8') as testf:
+            jscode = testf.read()
+        func = ie._parse_sig_js(jscode)
         src_sig = (
             compat_str(string.printable[:sig_input])
             if isinstance(sig_input, int) else sig_input)
         got_sig = func(src_sig)
         self.assertEqual(got_sig, expected_sig)
 
-    test_func.__name__ = str('test_signature_' + stype + '_' + test_id)
+    test_func.__name__ = str('test_signature_js_' + test_id)
     setattr(TestSignature, test_func.__name__, test_func)
 
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c87e54e6b..346311d9b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -398,7 +398,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                      (?(1).+)?                                                # if we found the ID, everything can follow
                      $""" % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
     _PLAYER_INFO_RE = (
-        r'/(?P<id>[a-zA-Z0-9_-]{8,})/player_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?/base\.js$',
+        r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
+        r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
     )
     _SUBTITLE_FORMATS = ('srv1', 'srv2', 'srv3', 'ttml', 'vtt')
@@ -1237,6 +1238,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         funcname = self._search_regex(
             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
+             r'\bm=(?P<sig>[a-zA-Z0-9$]{2})\(decodeURIComponent\(h\.s\)\)',
+             r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2})\(decodeURIComponent\(c\)\)',
+             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
              r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
              # Obsolete patterns

From 311ebdd9a57e72116136a464fbc0fa8cad32db42 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 8 Feb 2021 15:46:32 +0700
Subject: [PATCH 201/860] [xhamster] Extract formats from xplayer settings and
 extract filesizes (closes #28114)

---
 youtube_dl/extractor/xhamster.py | 80 +++++++++++++++++++++++++++-----
 1 file changed, 68 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index 76aeaf9a4..f73b9778f 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -11,11 +11,14 @@ from ..utils import (
     dict_get,
     extract_attributes,
     ExtractorError,
+    float_or_none,
     int_or_none,
     parse_duration,
+    str_or_none,
     try_get,
     unified_strdate,
     url_or_none,
+    urljoin,
 )
 
 
@@ -146,36 +149,89 @@ class XHamsterIE(InfoExtractor):
             video = initials['videoModel']
             title = video['title']
             formats = []
-            for format_id, formats_dict in video['sources'].items():
+            format_urls = set()
+            format_sizes = {}
+            sources = try_get(video, lambda x: x['sources'], dict) or {}
+            for format_id, formats_dict in sources.items():
                 if not isinstance(formats_dict, dict):
                     continue
+                download_sources = try_get(sources, lambda x: x['download'], dict) or {}
+                for quality, format_dict in download_sources.items():
+                    if not isinstance(format_dict, dict):
+                        continue
+                    format_sizes[quality] = float_or_none(format_dict.get('size'))
                 for quality, format_item in formats_dict.items():
                     if format_id == 'download':
                         # Download link takes some time to be generated,
                         # skipping for now
                         continue
-                        if not isinstance(format_item, dict):
-                            continue
-                        format_url = format_item.get('link')
-                        filesize = int_or_none(
-                            format_item.get('size'), invscale=1000000)
-                    else:
-                        format_url = format_item
-                        filesize = None
+                    format_url = format_item
                     format_url = url_or_none(format_url)
-                    if not format_url:
+                    if not format_url or format_url in format_urls:
                         continue
+                    format_urls.add(format_url)
                     formats.append({
                         'format_id': '%s-%s' % (format_id, quality),
                         'url': format_url,
                         'ext': determine_ext(format_url, 'mp4'),
                         'height': get_height(quality),
-                        'filesize': filesize,
+                        'filesize': format_sizes.get(quality),
                         'http_headers': {
                             'Referer': urlh.geturl(),
                         },
                     })
-            self._sort_formats(formats)
+            xplayer_sources = try_get(
+                initials, lambda x: x['xplayerSettings']['sources'], dict)
+            if xplayer_sources:
+                hls_sources = xplayer_sources.get('hls')
+                if isinstance(hls_sources, dict):
+                    for hls_format_key in ('url', 'fallback'):
+                        hls_url = hls_sources.get(hls_format_key)
+                        if not hls_url:
+                            continue
+                        hls_url = urljoin(url, hls_url)
+                        if not hls_url or hls_url in format_urls:
+                            continue
+                        format_urls.add(hls_url)
+                        formats.extend(self._extract_m3u8_formats(
+                            hls_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                            m3u8_id='hls', fatal=False))
+                standard_sources = xplayer_sources.get('standard')
+                if isinstance(standard_sources, dict):
+                    for format_id, formats_list in standard_sources.items():
+                        if not isinstance(formats_list, list):
+                            continue
+                        for standard_format in formats_list:
+                            if not isinstance(standard_format, dict):
+                                continue
+                            for standard_format_key in ('url', 'fallback'):
+                                standard_url = standard_format.get(standard_format_key)
+                                if not standard_url:
+                                    continue
+                                standard_url = urljoin(url, standard_url)
+                                if not standard_url or standard_url in format_urls:
+                                    continue
+                                format_urls.add(standard_url)
+                                ext = determine_ext(standard_url, 'mp4')
+                                if ext == 'm3u8':
+                                    formats.extend(self._extract_m3u8_formats(
+                                        standard_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                                        m3u8_id='hls', fatal=False))
+                                    continue
+                                quality = (str_or_none(standard_format.get('quality'))
+                                           or str_or_none(standard_format.get('label'))
+                                           or '')
+                                formats.append({
+                                    'format_id': '%s-%s' % (format_id, quality),
+                                    'url': standard_url,
+                                    'ext': ext,
+                                    'height': get_height(quality),
+                                    'filesize': format_sizes.get(quality),
+                                    'http_headers': {
+                                        'Referer': standard_url,
+                                    },
+                                })
+            self._sort_formats(formats, field_preference=('height', 'width', 'tbr', 'format_id'))
 
             categories_list = video.get('categories')
             if isinstance(categories_list, list):

From 7f8b8bc418b8831ea1c2ae8de64e3bf0e8b707f8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 8 Feb 2021 15:56:42 +0100
Subject: [PATCH 202/860] [ign] fix extraction(closes #24771)

---
 youtube_dl/extractor/extractors.py |   4 +-
 youtube_dl/extractor/ign.py        | 371 +++++++++++++++--------------
 2 files changed, 200 insertions(+), 175 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 97b0b4034..84998316c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -470,8 +470,8 @@ from .hungama import (
 from .hypem import HypemIE
 from .ign import (
     IGNIE,
-    OneUPIE,
-    PCMagIE,
+    IGNVideoIE,
+    IGNArticleIE,
 )
 from .iheart import (
     IHeartRadioIE,
diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index a96ea8010..0d9f50ed2 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -3,230 +3,255 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_parse_qs,
+    compat_urllib_parse_urlparse,
+)
 from ..utils import (
+    HEADRequest,
+    determine_ext,
     int_or_none,
     parse_iso8601,
+    strip_or_none,
+    try_get,
 )
 
 
-class IGNIE(InfoExtractor):
+class IGNBaseIE(InfoExtractor):
+    def _call_api(self, slug):
+        return self._download_json(
+            'http://apis.ign.com/{0}/v3/{0}s/slug/{1}'.format(self._PAGE_TYPE, slug), slug)
+
+
+class IGNIE(IGNBaseIE):
     """
     Extractor for some of the IGN sites, like www.ign.com, es.ign.com de.ign.com.
     Some videos of it.ign.com are also supported
     """
 
-    _VALID_URL = r'https?://.+?\.ign\.com/(?:[^/]+/)?(?P<type>videos|show_videos|articles|feature|(?:[^/]+/\d+/video))(/.+)?/(?P<name_or_id>.+)'
+    _VALID_URL = r'https?://(?:.+?\.ign|www\.pcmag)\.com/videos/(?:\d{4}/\d{2}/\d{2}/)?(?P<id>[^/?&#]+)'
     IE_NAME = 'ign.com'
+    _PAGE_TYPE = 'video'
 
-    _API_URL_TEMPLATE = 'http://apis.ign.com/video/v3/videos/%s'
-    _EMBED_RE = r'<iframe[^>]+?["\']((?:https?:)?//.+?\.ign\.com.+?/embed.+?)["\']'
-
-    _TESTS = [
-        {
-            'url': 'http://www.ign.com/videos/2013/06/05/the-last-of-us-review',
-            'md5': 'febda82c4bafecd2d44b6e1a18a595f8',
-            'info_dict': {
-                'id': '8f862beef863986b2785559b9e1aa599',
-                'ext': 'mp4',
-                'title': 'The Last of Us Review',
-                'description': 'md5:c8946d4260a4d43a00d5ae8ed998870c',
-                'timestamp': 1370440800,
-                'upload_date': '20130605',
-                'uploader_id': 'cberidon@ign.com',
-            }
-        },
-        {
-            'url': 'http://me.ign.com/en/feature/15775/100-little-things-in-gta-5-that-will-blow-your-mind',
-            'info_dict': {
-                'id': '100-little-things-in-gta-5-that-will-blow-your-mind',
-            },
-            'playlist': [
-                {
-                    'info_dict': {
-                        'id': '5ebbd138523268b93c9141af17bec937',
-                        'ext': 'mp4',
-                        'title': 'GTA 5 Video Review',
-                        'description': 'Rockstar drops the mic on this generation of games. Watch our review of the masterly Grand Theft Auto V.',
-                        'timestamp': 1379339880,
-                        'upload_date': '20130916',
-                        'uploader_id': 'danieljkrupa@gmail.com',
-                    },
-                },
-                {
-                    'info_dict': {
-                        'id': '638672ee848ae4ff108df2a296418ee2',
-                        'ext': 'mp4',
-                        'title': '26 Twisted Moments from GTA 5 in Slow Motion',
-                        'description': 'The twisted beauty of GTA 5 in stunning slow motion.',
-                        'timestamp': 1386878820,
-                        'upload_date': '20131212',
-                        'uploader_id': 'togilvie@ign.com',
-                    },
-                },
-            ],
-            'params': {
-                'skip_download': True,
-            },
-        },
-        {
-            'url': 'http://www.ign.com/articles/2014/08/15/rewind-theater-wild-trailer-gamescom-2014?watch',
-            'md5': '618fedb9c901fd086f6f093564ef8558',
-            'info_dict': {
-                'id': '078fdd005f6d3c02f63d795faa1b984f',
-                'ext': 'mp4',
-                'title': 'Rewind Theater - Wild Trailer Gamescom 2014',
-                'description': 'Brian and Jared explore Michel Ancel\'s captivating new preview.',
-                'timestamp': 1408047180,
-                'upload_date': '20140814',
-                'uploader_id': 'jamesduggan1990@gmail.com',
-            },
-        },
-        {
-            'url': 'http://me.ign.com/en/videos/112203/video/how-hitman-aims-to-be-different-than-every-other-s',
-            'only_matching': True,
-        },
-        {
-            'url': 'http://me.ign.com/ar/angry-birds-2/106533/video/lrd-ldyy-lwl-lfylm-angry-birds',
-            'only_matching': True,
-        },
-        {
-            # videoId pattern
-            'url': 'http://www.ign.com/articles/2017/06/08/new-ducktales-short-donalds-birthday-doesnt-go-as-planned',
-            'only_matching': True,
-        },
-    ]
-
-    def _find_video_id(self, webpage):
-        res_id = [
-            r'"video_id"\s*:\s*"(.*?)"',
-            r'class="hero-poster[^"]*?"[^>]*id="(.+?)"',
-            r'data-video-id="(.+?)"',
-            r'<object id="vid_(.+?)"',
-            r'<meta name="og:image" content=".*/(.+?)-(.+?)/.+.jpg"',
-            r'videoId&quot;\s*:\s*&quot;(.+?)&quot;',
-            r'videoId["\']\s*:\s*["\']([^"\']+?)["\']',
-        ]
-        return self._search_regex(res_id, webpage, 'video id', default=None)
+    _TESTS = [{
+        'url': 'http://www.ign.com/videos/2013/06/05/the-last-of-us-review',
+        'md5': 'd2e1586d9987d40fad7867bf96a018ea',
+        'info_dict': {
+            'id': '8f862beef863986b2785559b9e1aa599',
+            'ext': 'mp4',
+            'title': 'The Last of Us Review',
+            'description': 'md5:c8946d4260a4d43a00d5ae8ed998870c',
+            'timestamp': 1370440800,
+            'upload_date': '20130605',
+            'tags': 'count:9',
+        }
+    }, {
+        'url': 'http://www.pcmag.com/videos/2015/01/06/010615-whats-new-now-is-gogo-snooping-on-your-data',
+        'md5': 'f1581a6fe8c5121be5b807684aeac3f6',
+        'info_dict': {
+            'id': 'ee10d774b508c9b8ec07e763b9125b91',
+            'ext': 'mp4',
+            'title': 'What\'s New Now: Is GoGo Snooping on Your Data?',
+            'description': 'md5:817a20299de610bd56f13175386da6fa',
+            'timestamp': 1420571160,
+            'upload_date': '20150106',
+            'tags': 'count:4',
+        }
+    }, {
+        'url': 'https://www.ign.com/videos/is-a-resident-evil-4-remake-on-the-way-ign-daily-fix',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        name_or_id = mobj.group('name_or_id')
-        page_type = mobj.group('type')
-        webpage = self._download_webpage(url, name_or_id)
-        if page_type != 'video':
-            multiple_urls = re.findall(
-                r'<param name="flashvars"[^>]*value="[^"]*?url=(https?://www\.ign\.com/videos/.*?)["&]',
-                webpage)
-            if multiple_urls:
-                entries = [self.url_result(u, ie='IGN') for u in multiple_urls]
-                return {
-                    '_type': 'playlist',
-                    'id': name_or_id,
-                    'entries': entries,
-                }
-
-        video_id = self._find_video_id(webpage)
-        if not video_id:
-            return self.url_result(self._search_regex(
-                self._EMBED_RE, webpage, 'embed url'))
-        return self._get_video_info(video_id)
-
-    def _get_video_info(self, video_id):
-        api_data = self._download_json(
-            self._API_URL_TEMPLATE % video_id, video_id)
+        display_id = self._match_id(url)
+        video = self._call_api(display_id)
+        video_id = video['videoId']
+        metadata = video['metadata']
+        title = metadata.get('longTitle') or metadata.get('title') or metadata['name']
 
         formats = []
-        m3u8_url = api_data['refs'].get('m3uUrl')
+        refs = video.get('refs') or {}
+
+        m3u8_url = refs.get('m3uUrl')
         if m3u8_url:
             formats.extend(self._extract_m3u8_formats(
                 m3u8_url, video_id, 'mp4', 'm3u8_native',
                 m3u8_id='hls', fatal=False))
-        f4m_url = api_data['refs'].get('f4mUrl')
+
+        f4m_url = refs.get('f4mUrl')
         if f4m_url:
             formats.extend(self._extract_f4m_formats(
                 f4m_url, video_id, f4m_id='hds', fatal=False))
-        for asset in api_data['assets']:
+
+        for asset in (video.get('assets') or []):
+            asset_url = asset.get('url')
+            if not asset_url:
+                continue
             formats.append({
-                'url': asset['url'],
-                'tbr': asset.get('actual_bitrate_kbps'),
-                'fps': asset.get('frame_rate'),
+                'url': asset_url,
+                'tbr': int_or_none(asset.get('bitrate'), 1000),
+                'fps': int_or_none(asset.get('frame_rate')),
                 'height': int_or_none(asset.get('height')),
                 'width': int_or_none(asset.get('width')),
             })
+
+        mezzanine_url = try_get(video, lambda x: x['system']['mezzanineUrl'])
+        if mezzanine_url:
+            formats.append({
+                'ext': determine_ext(mezzanine_url, 'mp4'),
+                'format_id': 'mezzanine',
+                'preference': 1,
+                'url': mezzanine_url,
+            })
+
         self._sort_formats(formats)
 
-        thumbnails = [{
-            'url': thumbnail['url']
-        } for thumbnail in api_data.get('thumbnails', [])]
+        thumbnails = []
+        for thumbnail in (video.get('thumbnails') or []):
+            thumbnail_url = thumbnail.get('url')
+            if not thumbnail_url:
+                continue
+            thumbnails.append({
+                'url': thumbnail_url,
+            })
 
-        metadata = api_data['metadata']
+        tags = []
+        for tag in (video.get('tags') or []):
+            display_name = tag.get('displayName')
+            if not display_name:
+                continue
+            tags.append(display_name)
 
         return {
-            'id': api_data.get('videoId') or video_id,
-            'title': metadata.get('longTitle') or metadata.get('name') or metadata.get['title'],
-            'description': metadata.get('description'),
+            'id': video_id,
+            'title': title,
+            'description': strip_or_none(metadata.get('description')),
             'timestamp': parse_iso8601(metadata.get('publishDate')),
             'duration': int_or_none(metadata.get('duration')),
-            'display_id': metadata.get('slug') or video_id,
-            'uploader_id': metadata.get('creator'),
+            'display_id': display_id,
             'thumbnails': thumbnails,
             'formats': formats,
+            'tags': tags,
         }
 
 
-class OneUPIE(IGNIE):
-    _VALID_URL = r'https?://gamevideos\.1up\.com/(?P<type>video)/id/(?P<name_or_id>.+)\.html'
-    IE_NAME = '1up.com'
-
+class IGNVideoIE(InfoExtractor):
+    _VALID_URL = r'https?://.+?\.ign\.com/(?:[a-z]{2}/)?[^/]+/(?P<id>\d+)/(?:video|trailer)/'
     _TESTS = [{
-        'url': 'http://gamevideos.1up.com/video/id/34976.html',
-        'md5': 'c9cc69e07acb675c31a16719f909e347',
+        'url': 'http://me.ign.com/en/videos/112203/video/how-hitman-aims-to-be-different-than-every-other-s',
+        'md5': 'dd9aca7ed2657c4e118d8b261e5e9de1',
         'info_dict': {
-            'id': '34976',
+            'id': 'e9be7ea899a9bbfc0674accc22a36cc8',
             'ext': 'mp4',
-            'title': 'Sniper Elite V2 - Trailer',
-            'description': 'md5:bf0516c5ee32a3217aa703e9b1bc7826',
-            'timestamp': 1313099220,
-            'upload_date': '20110811',
-            'uploader_id': 'IGN',
+            'title': 'How Hitman Aims to Be Different Than Every Other Stealth Game - NYCC 2015',
+            'description': 'Taking out assassination targets in Hitman has never been more stylish.',
+            'timestamp': 1444665600,
+            'upload_date': '20151012',
         }
+    }, {
+        'url': 'http://me.ign.com/ar/angry-birds-2/106533/video/lrd-ldyy-lwl-lfylm-angry-birds',
+        'only_matching': True,
+    }, {
+        # Youtube embed
+        'url': 'https://me.ign.com/ar/ratchet-clank-rift-apart/144327/trailer/embed',
+        'only_matching': True,
+    }, {
+        # Twitter embed
+        'url': 'http://adria.ign.com/sherlock-season-4/9687/trailer/embed',
+        'only_matching': True,
+    }, {
+        # Vimeo embed
+        'url': 'https://kr.ign.com/bic-2018/3307/trailer/embed',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        result = super(OneUPIE, self)._real_extract(url)
-        result['id'] = mobj.group('name_or_id')
-        return result
+        video_id = self._match_id(url)
+        req = HEADRequest(url.rsplit('/', 1)[0] + '/embed')
+        url = self._request_webpage(req, video_id).geturl()
+        ign_url = compat_parse_qs(
+            compat_urllib_parse_urlparse(url).query).get('url', [None])[0]
+        if ign_url:
+            return self.url_result(ign_url, IGNIE.ie_key())
+        return self.url_result(url)
 
 
-class PCMagIE(IGNIE):
-    _VALID_URL = r'https?://(?:www\.)?pcmag\.com/(?P<type>videos|article2)(/.+)?/(?P<name_or_id>.+)'
-    IE_NAME = 'pcmag'
-
-    _EMBED_RE = r'iframe\.setAttribute\("src",\s*__util.objToUrlString\("http://widgets\.ign\.com/video/embed/content\.html?[^"]*url=([^"]+)["&]'
-
+class IGNArticleIE(IGNBaseIE):
+    _VALID_URL = r'https?://.+?\.ign\.com/(?:articles(?:/\d{4}/\d{2}/\d{2})?|(?:[a-z]{2}/)?feature/\d+)/(?P<id>[^/?&#]+)'
+    _PAGE_TYPE = 'article'
     _TESTS = [{
-        'url': 'http://www.pcmag.com/videos/2015/01/06/010615-whats-new-now-is-gogo-snooping-on-your-data',
-        'md5': '212d6154fd0361a2781075f1febbe9ad',
+        'url': 'http://me.ign.com/en/feature/15775/100-little-things-in-gta-5-that-will-blow-your-mind',
         'info_dict': {
-            'id': 'ee10d774b508c9b8ec07e763b9125b91',
-            'ext': 'mp4',
-            'title': '010615_What\'s New Now: Is GoGo Snooping on Your Data?',
-            'description': 'md5:a7071ae64d2f68cc821c729d4ded6bb3',
-            'timestamp': 1420571160,
-            'upload_date': '20150106',
-            'uploader_id': 'cozzipix@gmail.com',
-        }
+            'id': '524497489e4e8ff5848ece34',
+            'title': '100 Little Things in GTA 5 That Will Blow Your Mind',
+        },
+        'playlist': [
+            {
+                'info_dict': {
+                    'id': '5ebbd138523268b93c9141af17bec937',
+                    'ext': 'mp4',
+                    'title': 'GTA 5 Video Review',
+                    'description': 'Rockstar drops the mic on this generation of games. Watch our review of the masterly Grand Theft Auto V.',
+                    'timestamp': 1379339880,
+                    'upload_date': '20130916',
+                },
+            },
+            {
+                'info_dict': {
+                    'id': '638672ee848ae4ff108df2a296418ee2',
+                    'ext': 'mp4',
+                    'title': '26 Twisted Moments from GTA 5 in Slow Motion',
+                    'description': 'The twisted beauty of GTA 5 in stunning slow motion.',
+                    'timestamp': 1386878820,
+                    'upload_date': '20131212',
+                },
+            },
+        ],
+        'params': {
+            'playlist_items': '2-3',
+            'skip_download': True,
+        },
     }, {
-        'url': 'http://www.pcmag.com/article2/0,2817,2470156,00.asp',
-        'md5': '94130c1ca07ba0adb6088350681f16c1',
+        'url': 'http://www.ign.com/articles/2014/08/15/rewind-theater-wild-trailer-gamescom-2014?watch',
         'info_dict': {
-            'id': '042e560ba94823d43afcb12ddf7142ca',
-            'ext': 'mp4',
-            'title': 'HTC\'s Weird New Re Camera - What\'s New Now',
-            'description': 'md5:53433c45df96d2ea5d0fda18be2ca908',
-            'timestamp': 1412953920,
-            'upload_date': '20141010',
-            'uploader_id': 'chris_snyder@pcmag.com',
-        }
+            'id': '53ee806780a81ec46e0790f8',
+            'title': 'Rewind Theater - Wild Trailer Gamescom 2014',
+        },
+        'playlist_count': 2,
+    }, {
+        # videoId pattern
+        'url': 'http://www.ign.com/articles/2017/06/08/new-ducktales-short-donalds-birthday-doesnt-go-as-planned',
+        'only_matching': True,
+    }, {
+        # Youtube embed
+        'url': 'https://www.ign.com/articles/2021-mvp-named-in-puppy-bowl-xvii',
+        'only_matching': True,
+    }, {
+        # IMDB embed
+        'url': 'https://www.ign.com/articles/2014/08/07/sons-of-anarchy-final-season-trailer',
+        'only_matching': True,
+    }, {
+        # Facebook embed
+        'url': 'https://www.ign.com/articles/2017/09/20/marvels-the-punisher-watch-the-new-trailer-for-the-netflix-series',
+        'only_matching': True,
+    }, {
+        # Brightcove embed
+        'url': 'https://www.ign.com/articles/2016/01/16/supergirl-goes-flying-with-martian-manhunter-in-new-clip',
+        'only_matching': True,
     }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        article = self._call_api(display_id)
+
+        def entries():
+            media_url = try_get(article, lambda x: x['mediaRelations'][0]['media']['metadata']['url'])
+            if media_url:
+                yield self.url_result(media_url, IGNIE.ie_key())
+            for content in (article.get('content') or []):
+                for video_url in re.findall(r'(?:\[(?:ignvideo\s+url|youtube\s+clip_id)|<iframe[^>]+src)="([^"]+)"', content):
+                    yield self.url_result(video_url)
+
+        return self.playlist_result(
+            entries(), article.get('articleId'),
+            strip_or_none(try_get(article, lambda x: x['metadata']['headline'])))

From a4c7ed6b1e9100be8ef65c44e7e6e43b9314ff5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Feb 2021 22:28:58 +0700
Subject: [PATCH 203/860] [youtube:tab] Improve grid continuation extraction
 (closes #28130)

---
 youtube_dl/extractor/youtube.py | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 346311d9b..c78996629 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2374,9 +2374,9 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
         next_continuation = cls._extract_next_continuation_data(renderer)
         if next_continuation:
             return next_continuation
-        contents = renderer.get('contents')
-        if not isinstance(contents, list):
-            return
+        contents = []
+        for key in ('contents', 'items'):
+            contents.extend(try_get(renderer, lambda x: x[key], list) or [])
         for content in contents:
             if not isinstance(content, dict):
                 continue
@@ -2509,6 +2509,13 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                 continuation_item = continuation_items[0]
                 if not isinstance(continuation_item, dict):
                     continue
+                renderer = continuation_item.get('gridVideoRenderer')
+                if renderer:
+                    grid_renderer = {'items': continuation_items}
+                    for entry in self._grid_entries(grid_renderer):
+                        yield entry
+                    continuation = self._extract_continuation(grid_renderer)
+                    continue
                 renderer = continuation_item.get('playlistVideoRenderer') or continuation_item.get('itemSectionRenderer')
                 if renderer:
                     video_list_renderer = {'contents': continuation_items}

From cd493c5adcb526cdfa2a9d5194269b671a0dc343 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Feb 2021 22:32:25 +0700
Subject: [PATCH 204/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 784b73d8d..5951372b3 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,20 @@
+version <unreleased>
+
+Extractors
+* [youtube:tab] Improve grid continuation extraction (#28130)
+* [ign] Fix extraction (#24771)
++ [xhamster] Extract format filesize
++ [xhamster] Extract formats from xplayer settings (#28114)
++ [youtube] Add support phone/tablet JS player (#26424)
+* [archiveorg] Fix and improve extraction (#21330, #23586, #25277, #26780,
+  #27109, #27236, #28063)
++ [cda] Detect geo restricted videos (#28106)
+* [urplay] Fix extraction (#28073, #28074)
+* [youtube] Fix release date extraction (#28094)
++ [youtube] Extract abr and vbr (#28100)
+* [youtube] Skip OTF formats (#28070)
+
+
 version 2021.02.04.1
 
 Extractors

From 360d5f0daac879a1371c6a45e0d3310ced60e352 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Feb 2021 22:34:47 +0700
Subject: [PATCH 205/860] release 2021.02.10

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 docs/supportedsites.md                           | 4 ++--
 youtube_dl/version.py                            | 2 +-
 8 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 19b750f86..ea0a59dca 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.10. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.02.04.1**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.10**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.02.04.1
+ [debug] youtube-dl version 2021.02.10
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index 8acb80b60..d24855c72 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.10. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.02.04.1**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.10**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 66edcf752..8b96a2883 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.10. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.02.04.1**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.10**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index 18203fb34..e46971047 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.10. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.02.04.1**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.10**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.02.04.1
+ [debug] youtube-dl version 2021.02.10
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index 20df40cc5..a9ca379ca 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.04.1. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.10. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.02.04.1**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.10**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index 5951372b3..384bd19c2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.02.10
 
 Extractors
 * [youtube:tab] Improve grid continuation extraction (#28130)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index e1b85b1d1..1373cc4f6 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -1,6 +1,5 @@
 # Supported sites
  - **1tv**: Первый канал
- - **1up.com**
  - **20min**
  - **220.ro**
  - **23video**
@@ -376,6 +375,8 @@
  - **HungamaSong**
  - **Hypem**
  - **ign.com**
+ - **IGNArticle**
+ - **IGNVideo**
  - **IHeartRadio**
  - **iheartradio:podcast**
  - **imdb**: Internet Movie Database trailers
@@ -676,7 +677,6 @@
  - **parliamentlive.tv**: UK parliament videos
  - **Patreon**
  - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET  (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC)
- - **pcmag**
  - **PearVideo**
  - **PeerTube**
  - **People**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 425f15589..79d2be625 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.02.04.1'
+__version__ = '2021.02.10'

From f28f1b4d6ed053f1dbfbc7fc992162c4ea4f2ce7 Mon Sep 17 00:00:00 2001
From: Kevin Velghe <kevin@paretje.be>
Date: Thu, 11 Feb 2021 09:04:16 +0100
Subject: [PATCH 206/860] [canvas] Add new extractor for Dagelijkse Kost
 (#28119)

---
 youtube_dl/extractor/canvas.py     | 56 ++++++++++++++++++++++++++++--
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 54 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/canvas.py b/youtube_dl/extractor/canvas.py
index 8b76a0200..eefbab241 100644
--- a/youtube_dl/extractor/canvas.py
+++ b/youtube_dl/extractor/canvas.py
@@ -7,19 +7,21 @@ from .common import InfoExtractor
 from .gigya import GigyaBaseIE
 from ..compat import compat_HTTPError
 from ..utils import (
-    extract_attributes,
     ExtractorError,
-    strip_or_none,
+    clean_html,
+    extract_attributes,
     float_or_none,
+    get_element_by_class,
     int_or_none,
     merge_dicts,
     str_or_none,
+    strip_or_none,
     url_or_none,
 )
 
 
 class CanvasIE(InfoExtractor):
-    _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrt(?:video|nieuws)|sporza)/assets/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://mediazone\.vrt\.be/api/v1/(?P<site_id>canvas|een|ketnet|vrt(?:video|nieuws)|sporza|dako)/assets/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://mediazone.vrt.be/api/v1/ketnet/assets/md-ast-4ac54990-ce66-4d00-a8ca-9eac86f4c475',
         'md5': '68993eda72ef62386a15ea2cf3c93107',
@@ -332,3 +334,51 @@ class VrtNUIE(GigyaBaseIE):
             'display_id': display_id,
             'season_number': int_or_none(page.get('episode_season')),
         })
+
+
+class DagelijkseKostIE(InfoExtractor):
+    IE_DESC = 'dagelijksekost.een.be'
+    _VALID_URL = r'https?://dagelijksekost\.een\.be/gerechten/(?P<id>[^/?#&]+)'
+    _TEST = {
+        'url': 'https://dagelijksekost.een.be/gerechten/hachis-parmentier-met-witloof',
+        'md5': '30bfffc323009a3e5f689bef6efa2365',
+        'info_dict': {
+            'id': 'md-ast-27a4d1ff-7d7b-425e-b84f-a4d227f592fa',
+            'display_id': 'hachis-parmentier-met-witloof',
+            'ext': 'mp4',
+            'title': 'Hachis parmentier met witloof',
+            'description': 'md5:9960478392d87f63567b5b117688cdc5',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 283.02,
+        },
+        'expected_warnings': ['is not a supported codec'],
+    }
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        title = strip_or_none(get_element_by_class(
+            'dish-metadata__title', webpage
+        ) or self._html_search_meta(
+            'twitter:title', webpage))
+
+        description = clean_html(get_element_by_class(
+            'dish-description', webpage)
+        ) or self._html_search_meta(
+            ('description', 'twitter:description', 'og:description'),
+            webpage)
+
+        video_id = self._html_search_regex(
+            r'data-url=(["\'])(?P<id>(?:(?!\1).)+)\1', webpage, 'video id',
+            group='id')
+
+        return {
+            '_type': 'url_transparent',
+            'url': 'https://mediazone.vrt.be/api/v1/dako/assets/%s' % video_id,
+            'ie_key': CanvasIE.ie_key(),
+            'id': video_id,
+            'display_id': display_id,
+            'title': title,
+            'description': description,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 84998316c..e4c475fd8 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -163,6 +163,7 @@ from .canvas import (
     CanvasIE,
     CanvasEenIE,
     VrtNUIE,
+    DagelijkseKostIE,
 )
 from .carambatv import (
     CarambaTVIE,

From f94d76499362017f673520286bc3848916735275 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Feb 2021 05:03:15 +0700
Subject: [PATCH 207/860] [ard] Improve formats extraction (closes #28155)

---
 youtube_dl/extractor/ard.py | 44 +++++++++++++++++++++++++++----------
 1 file changed, 33 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 6bf5b3f13..143fc51e9 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -284,20 +284,42 @@ class ARDIE(InfoExtractor):
 
         formats = []
         for a in video_node.findall('.//asset'):
+            file_name = xpath_text(a, './fileName', default=None)
+            if not file_name:
+                continue
+            format_type = a.attrib.get('type')
+            format_url = url_or_none(file_name)
+            if format_url:
+                ext = determine_ext(file_name)
+                if ext == 'm3u8':
+                    formats.extend(self._extract_m3u8_formats(
+                        format_url, display_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id=format_type or 'hls', fatal=False))
+                    continue
+                elif ext == 'f4m':
+                    formats.extend(self._extract_f4m_formats(
+                        update_url_query(format_url, {'hdcore': '3.7.0'}),
+                        display_id, f4m_id=format_type or 'hds', fatal=False))
+                    continue
             f = {
-                'format_id': a.attrib['type'],
-                'width': int_or_none(a.find('./frameWidth').text),
-                'height': int_or_none(a.find('./frameHeight').text),
-                'vbr': int_or_none(a.find('./bitrateVideo').text),
-                'abr': int_or_none(a.find('./bitrateAudio').text),
-                'vcodec': a.find('./codecVideo').text,
-                'tbr': int_or_none(a.find('./totalBitrate').text),
+                'format_id': format_type,
+                'width': int_or_none(xpath_text(a, './frameWidth')),
+                'height': int_or_none(xpath_text(a, './frameHeight')),
+                'vbr': int_or_none(xpath_text(a, './bitrateVideo')),
+                'abr': int_or_none(xpath_text(a, './bitrateAudio')),
+                'vcodec': xpath_text(a, './codecVideo'),
+                'tbr': int_or_none(xpath_text(a, './totalBitrate')),
             }
-            if a.find('./serverPrefix').text:
-                f['url'] = a.find('./serverPrefix').text
-                f['playpath'] = a.find('./fileName').text
+            server_prefix = xpath_text(a, './serverPrefix', default=None)
+            if server_prefix:
+                f.update({
+                    'url': server_prefix,
+                    'playpath': file_name,
+                })
             else:
-                f['url'] = a.find('./fileName').text
+                if not format_url:
+                    continue
+                f['url'] = format_url
             formats.append(f)
         self._sort_formats(formats)
 

From 6d32c6c6d3b0588b193eaeb4178592219c3b4df8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 14 Feb 2021 16:22:45 +0100
Subject: [PATCH 208/860] [xboxclips] fix extraction(closes #27151)

---
 youtube_dl/extractor/xboxclips.py | 45 ++++++++++++++++++++-----------
 1 file changed, 30 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/xboxclips.py b/youtube_dl/extractor/xboxclips.py
index d9c277bc3..25f487e1e 100644
--- a/youtube_dl/extractor/xboxclips.py
+++ b/youtube_dl/extractor/xboxclips.py
@@ -1,40 +1,55 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..compat import (
+    compat_parse_qs,
+    compat_urllib_parse_urlparse,
+)
 from ..utils import (
     int_or_none,
+    month_by_abbreviation,
     parse_filesize,
-    unified_strdate,
 )
 
 
 class XboxClipsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?xboxclips\.com/(?:video\.php\?.*vid=|[^/]+/)(?P<id>[\w-]{36})'
-    _TEST = {
+    _VALID_URL = r'https?://(?:www\.)?(?:xboxclips\.com|gameclips\.io)/(?:video\.php\?.*vid=|[^/]+/)(?P<id>[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12})'
+    _TESTS = [{
         'url': 'http://xboxclips.com/video.php?uid=2533274823424419&gamertag=Iabdulelah&vid=074a69a9-5faf-46aa-b93b-9909c1720325',
         'md5': 'fbe1ec805e920aeb8eced3c3e657df5d',
         'info_dict': {
             'id': '074a69a9-5faf-46aa-b93b-9909c1720325',
             'ext': 'mp4',
-            'title': 'Iabdulelah playing Titanfall',
+            'title': 'iAbdulElah playing Titanfall',
             'filesize_approx': 26800000,
             'upload_date': '20140807',
             'duration': 56,
         }
-    }
+    }, {
+        'url': 'https://gameclips.io/iAbdulElah/074a69a9-5faf-46aa-b93b-9909c1720325',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        if '/video.php' in url:
+            qs = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+            url = 'https://gameclips.io/%s/%s' % (qs['gamertag'][0], qs['vid'][0])
 
-        video_url = self._html_search_regex(
-            r'>(?:Link|Download): <a[^>]+href="([^"]+)"', webpage, 'video URL')
-        title = self._html_search_regex(
-            r'<title>XboxClips \| ([^<]+)</title>', webpage, 'title')
-        upload_date = unified_strdate(self._html_search_regex(
-            r'>Recorded: ([^<]+)<', webpage, 'upload date', fatal=False))
+        webpage = self._download_webpage(url, video_id)
+        info = self._parse_html5_media_entries(url, webpage, video_id)[0]
+
+        title = self._html_search_meta(['og:title', 'twitter:title'], webpage)
+        upload_date = None
+        mobj = re.search(
+            r'>Recorded: (\d{2})-(Jan|Feb|Mar|Apr|May|Ju[nl]|Aug|Sep|Oct|Nov|Dec)-(\d{4})',
+            webpage)
+        if mobj:
+            upload_date = '%s%.2d%s' % (mobj.group(3), month_by_abbreviation(mobj.group(2)), mobj.group(1))
         filesize = parse_filesize(self._html_search_regex(
             r'>Size: ([^<]+)<', webpage, 'file size', fatal=False))
         duration = int_or_none(self._html_search_regex(
@@ -42,12 +57,12 @@ class XboxClipsIE(InfoExtractor):
         view_count = int_or_none(self._html_search_regex(
             r'>Views: (\d+)<', webpage, 'view count', fatal=False))
 
-        return {
+        info.update({
             'id': video_id,
-            'url': video_url,
             'title': title,
             'upload_date': upload_date,
             'filesize_approx': filesize,
             'duration': duration,
             'view_count': view_count,
-        }
+        })
+        return info

From d8085580f63ad3b146a31712ff76cf41d5a4558a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 14 Feb 2021 19:48:26 +0100
Subject: [PATCH 209/860] [kakao] improve info extraction and detect geo
 restriction(closes #26577)

---
 youtube_dl/extractor/kakao.py | 64 ++++++++++++++++-------------------
 1 file changed, 30 insertions(+), 34 deletions(-)

diff --git a/youtube_dl/extractor/kakao.py b/youtube_dl/extractor/kakao.py
index 32935bb28..31ce7a85c 100644
--- a/youtube_dl/extractor/kakao.py
+++ b/youtube_dl/extractor/kakao.py
@@ -3,10 +3,13 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..compat import compat_HTTPError
 from ..utils import (
+    ExtractorError,
     int_or_none,
+    str_or_none,
     strip_or_none,
+    try_get,
     unified_timestamp,
     update_url_query,
 )
@@ -23,7 +26,7 @@ class KakaoIE(InfoExtractor):
             'id': '301965083',
             'ext': 'mp4',
             'title': '乃木坂46 バナナマン 「3期生紹介コーナーが始動！顔高低差GPも！」 『乃木坂工事中』',
-            'uploader_id': 2671005,
+            'uploader_id': '2671005',
             'uploader': '그랑그랑이',
             'timestamp': 1488160199,
             'upload_date': '20170227',
@@ -36,11 +39,15 @@ class KakaoIE(InfoExtractor):
             'ext': 'mp4',
             'description': '러블리즈 - Destiny (나의 지구) (Lovelyz - Destiny)\r\n\r\n[쇼! 음악중심] 20160611, 507회',
             'title': '러블리즈 - Destiny (나의 지구) (Lovelyz - Destiny)',
-            'uploader_id': 2653210,
+            'uploader_id': '2653210',
             'uploader': '쇼! 음악중심',
             'timestamp': 1485684628,
             'upload_date': '20170129',
         }
+    }, {
+        # geo restricted
+        'url': 'https://tv.kakao.com/channel/3643855/cliplink/412069491',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -68,8 +75,7 @@ class KakaoIE(InfoExtractor):
             'fields': ','.join([
                 '-*', 'tid', 'clipLink', 'displayTitle', 'clip', 'title',
                 'description', 'channelId', 'createTime', 'duration', 'playCount',
-                'likeCount', 'commentCount', 'tagList', 'channel', 'name',
-                'clipChapterThumbnailList', 'thumbnailUrl', 'timeInSec', 'isDefault',
+                'likeCount', 'commentCount', 'tagList', 'channel', 'name', 'thumbnailUrl',
                 'videoOutputList', 'width', 'height', 'kbps', 'profile', 'label'])
         }
 
@@ -82,24 +88,28 @@ class KakaoIE(InfoExtractor):
 
         title = clip.get('title') or clip_link.get('displayTitle')
 
-        query['tid'] = impress.get('tid', '')
+        query.update({
+            'fields': '-*,code,message,url',
+            'tid': impress.get('tid') or '',
+        })
 
         formats = []
-        for fmt in clip.get('videoOutputList', []):
+        for fmt in (clip.get('videoOutputList') or []):
             try:
                 profile_name = fmt['profile']
                 if profile_name == 'AUDIO':
                     continue
-                query.update({
-                    'profile': profile_name,
-                    'fields': '-*,url',
-                })
-                fmt_url_json = self._download_json(
-                    api_base + 'raw/videolocation', display_id,
-                    'Downloading video URL for profile %s' % profile_name,
-                    query=query, headers=player_header, fatal=False)
-
-                if fmt_url_json is None:
+                query['profile'] = profile_name
+                try:
+                    fmt_url_json = self._download_json(
+                        api_base + 'raw/videolocation', display_id,
+                        'Downloading video URL for profile %s' % profile_name,
+                        query=query, headers=player_header)
+                except ExtractorError as e:
+                    if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
+                        resp = self._parse_json(e.cause.read().decode(), video_id)
+                        if resp.get('code') == 'GeoBlocked':
+                            self.raise_geo_restricted()
                     continue
 
                 fmt_url = fmt_url_json['url']
@@ -116,27 +126,13 @@ class KakaoIE(InfoExtractor):
                 pass
         self._sort_formats(formats)
 
-        thumbs = []
-        for thumb in clip.get('clipChapterThumbnailList', []):
-            thumbs.append({
-                'url': thumb.get('thumbnailUrl'),
-                'id': compat_str(thumb.get('timeInSec')),
-                'preference': -1 if thumb.get('isDefault') else 0
-            })
-        top_thumbnail = clip.get('thumbnailUrl')
-        if top_thumbnail:
-            thumbs.append({
-                'url': top_thumbnail,
-                'preference': 10,
-            })
-
         return {
             'id': display_id,
             'title': title,
             'description': strip_or_none(clip.get('description')),
-            'uploader': clip_link.get('channel', {}).get('name'),
-            'uploader_id': clip_link.get('channelId'),
-            'thumbnails': thumbs,
+            'uploader': try_get(clip_link, lambda x: x['channel']['name']),
+            'uploader_id': str_or_none(clip_link.get('channelId')),
+            'thumbnail': clip.get('thumbnailUrl'),
             'timestamp': unified_timestamp(clip_link.get('createTime')),
             'duration': int_or_none(clip.get('duration')),
             'view_count': int_or_none(clip.get('playCount')),

From be2e9b76eea73b073f00871ea831ee3f4a1000b3 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 14 Feb 2021 22:10:06 +0100
Subject: [PATCH 210/860] [videopress] add support for video.wordpress.com

---
 youtube_dl/extractor/videopress.py | 26 +++++++++++++++-----------
 1 file changed, 15 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/videopress.py b/youtube_dl/extractor/videopress.py
index e5f964d39..6376ff096 100644
--- a/youtube_dl/extractor/videopress.py
+++ b/youtube_dl/extractor/videopress.py
@@ -4,21 +4,22 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_str
 from ..utils import (
     determine_ext,
     float_or_none,
+    int_or_none,
     parse_age_limit,
     qualities,
     random_birthday,
-    try_get,
     unified_timestamp,
     urljoin,
 )
 
 
 class VideoPressIE(InfoExtractor):
-    _VALID_URL = r'https?://videopress\.com/embed/(?P<id>[\da-zA-Z]+)'
+    _ID_REGEX = r'[\da-zA-Z]{8}'
+    _PATH_REGEX = r'video(?:\.word)?press\.com/embed/'
+    _VALID_URL = r'https?://%s(?P<id>%s)' % (_PATH_REGEX, _ID_REGEX)
     _TESTS = [{
         'url': 'https://videopress.com/embed/kUJmAcSf',
         'md5': '706956a6c875873d51010921310e4bc6',
@@ -36,35 +37,36 @@ class VideoPressIE(InfoExtractor):
         # 17+, requires birth_* params
         'url': 'https://videopress.com/embed/iH3gstfZ',
         'only_matching': True,
+    }, {
+        'url': 'https://video.wordpress.com/embed/kUJmAcSf',
+        'only_matching': True,
     }]
 
     @staticmethod
     def _extract_urls(webpage):
         return re.findall(
-            r'<iframe[^>]+src=["\']((?:https?://)?videopress\.com/embed/[\da-zA-Z]+)',
+            r'<iframe[^>]+src=["\']((?:https?://)?%s%s)' % (VideoPressIE._PATH_REGEX, VideoPressIE._ID_REGEX),
             webpage)
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         query = random_birthday('birth_year', 'birth_month', 'birth_day')
+        query['fields'] = 'description,duration,file_url_base,files,height,original,poster,rating,title,upload_date,width'
         video = self._download_json(
             'https://public-api.wordpress.com/rest/v1.1/videos/%s' % video_id,
             video_id, query=query)
 
         title = video['title']
 
-        def base_url(scheme):
-            return try_get(
-                video, lambda x: x['file_url_base'][scheme], compat_str)
-
-        base_url = base_url('https') or base_url('http')
+        file_url_base = video.get('file_url_base') or {}
+        base_url = file_url_base.get('https') or file_url_base.get('http')
 
         QUALITIES = ('std', 'dvd', 'hd')
         quality = qualities(QUALITIES)
 
         formats = []
-        for format_id, f in video['files'].items():
+        for format_id, f in (video.get('files') or {}).items():
             if not isinstance(f, dict):
                 continue
             for ext, path in f.items():
@@ -75,12 +77,14 @@ class VideoPressIE(InfoExtractor):
                         'ext': determine_ext(path, ext),
                         'quality': quality(format_id),
                     })
-        original_url = try_get(video, lambda x: x['original'], compat_str)
+        original_url = video.get('original')
         if original_url:
             formats.append({
                 'url': original_url,
                 'format_id': 'original',
                 'quality': len(QUALITIES),
+                'width': int_or_none(video.get('width')),
+                'height': int_or_none(video.get('height')),
             })
         self._sort_formats(formats)
 

From 4b5410c5c841b826965ea76d62bea82a26a9e1b8 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 15 Feb 2021 13:06:54 +0100
Subject: [PATCH 211/860] [ccma] fix timestamp parsing in python 2

---
 youtube_dl/extractor/ccma.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/ccma.py b/youtube_dl/extractor/ccma.py
index 4db51e650..e6ae49352 100644
--- a/youtube_dl/extractor/ccma.py
+++ b/youtube_dl/extractor/ccma.py
@@ -1,12 +1,14 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import calendar
 import datetime
 import re
 
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
+    extract_timezone,
     int_or_none,
     parse_duration,
     parse_resolution,
@@ -97,8 +99,9 @@ class CCMAIE(InfoExtractor):
         timestamp = None
         data_utc = try_get(informacio, lambda x: x['data_emissio']['utc'])
         try:
-            timestamp = datetime.datetime.strptime(
-                data_utc, '%Y-%d-%mT%H:%M:%S%z').timestamp()
+            timezone, data_utc = extract_timezone(data_utc)
+            timestamp = calendar.timegm((datetime.datetime.strptime(
+                data_utc, '%Y-%d-%mT%H:%M:%S') - timezone).timetuple())
         except TypeError:
             pass
 

From 07eb8f19169c58bce0e784607ea350ae16ed5363 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 16 Feb 2021 05:57:53 +0700
Subject: [PATCH 212/860] [youtube] Fix controversial videos when authenticated
 with cookies (closes #28174)

---
 youtube_dl/extractor/youtube.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c78996629..7db4503e0 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1042,6 +1042,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # controversial video, only works with bpctr when authenticated with cookies
+            'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
+            'only_matching': True,
+        },
     ]
     _formats = {
         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
@@ -1405,7 +1410,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         video_id = self._match_id(url)
         base_url = self.http_scheme() + '//www.youtube.com/'
         webpage_url = base_url + 'watch?v=' + video_id
-        webpage = self._download_webpage(webpage_url, video_id, fatal=False)
+        webpage = self._download_webpage(
+            webpage_url + '&bpctr=9999999999', video_id, fatal=False)
 
         player_response = None
         if webpage:

From 56c63c8c02d7b9aabbced8d150badb6b520825d2 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 16 Feb 2021 10:08:43 +0100
Subject: [PATCH 213/860] [zhihu] Add new extractor(closes #28177)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/zhihu.py      | 69 ++++++++++++++++++++++++++++++
 2 files changed, 70 insertions(+)
 create mode 100644 youtube_dl/extractor/zhihu.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e4c475fd8..4347f1b74 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1603,5 +1603,6 @@ from .zattoo import (
     ZattooLiveIE,
 )
 from .zdf import ZDFIE, ZDFChannelIE
+from .zhihu import ZhihuIE
 from .zingmp3 import ZingMp3IE
 from .zype import ZypeIE
diff --git a/youtube_dl/extractor/zhihu.py b/youtube_dl/extractor/zhihu.py
new file mode 100644
index 000000000..d1ed55be3
--- /dev/null
+++ b/youtube_dl/extractor/zhihu.py
@@ -0,0 +1,69 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import float_or_none, int_or_none
+
+
+class ZhihuIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?zhihu\.com/zvideo/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://www.zhihu.com/zvideo/1342930761977176064',
+        'md5': 'c8d4c9cd72dd58e6f9bc9c2c84266464',
+        'info_dict': {
+            'id': '1342930761977176064',
+            'ext': 'mp4',
+            'title': '写春联也太难了吧！',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'uploader': '桥半舫',
+            'timestamp': 1612959715,
+            'upload_date': '20210210',
+            'uploader_id': '244ecb13b0fd7daf92235288c8ca3365',
+            'duration': 146.333,
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        zvideo = self._download_json(
+            'https://www.zhihu.com/api/v4/zvideos/' + video_id, video_id)
+        title = zvideo['title']
+        video = zvideo.get('video') or {}
+
+        formats = []
+        for format_id, q in (video.get('playlist') or {}).items():
+            play_url = q.get('url') or q.get('play_url')
+            if not play_url:
+                continue
+            formats.append({
+                'asr': int_or_none(q.get('sample_rate')),
+                'filesize': int_or_none(q.get('size')),
+                'format_id': format_id,
+                'fps': int_or_none(q.get('fps')),
+                'height': int_or_none(q.get('height')),
+                'tbr': float_or_none(q.get('bitrate')),
+                'url': play_url,
+                'width': int_or_none(q.get('width')),
+            })
+        self._sort_formats(formats)
+
+        author = zvideo.get('author') or {}
+        url_token = author.get('url_token')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': video.get('thumbnail') or zvideo.get('image_url'),
+            'uploader': author.get('name'),
+            'timestamp': int_or_none(zvideo.get('published_at')),
+            'uploader_id': author.get('id'),
+            'uploader_url': 'https://www.zhihu.com/people/' + url_token if url_token else None,
+            'duration': float_or_none(video.get('duration')),
+            'view_count': int_or_none(zvideo.get('play_count')),
+            'like_count': int_or_none(zvideo.get('liked_count')),
+            'comment_count': int_or_none(zvideo.get('comment_count')),
+        }

From 844e4cbc547f2a2f76053786522bdd6b53bf9ae1 Mon Sep 17 00:00:00 2001
From: Stephen Stair <sgstair@akkit.org>
Date: Sun, 16 Aug 2020 17:07:14 -0700
Subject: [PATCH 214/860] [storyfire] Add new extractor(closes #25628)(closes
 #26349)

---
 youtube_dl/extractor/extractors.py |   5 +
 youtube_dl/extractor/storyfire.py  | 151 +++++++++++++++++++++++++++++
 2 files changed, 156 insertions(+)
 create mode 100644 youtube_dl/extractor/storyfire.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 4347f1b74..51f6d38e9 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1141,6 +1141,11 @@ from .srgssr import (
 from .srmediathek import SRMediathekIE
 from .stanfordoc import StanfordOpenClassroomIE
 from .steam import SteamIE
+from .storyfire import (
+    StoryFireIE,
+    StoryFireUserIE,
+    StoryFireSeriesIE,
+)
 from .streamable import StreamableIE
 from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
diff --git a/youtube_dl/extractor/storyfire.py b/youtube_dl/extractor/storyfire.py
new file mode 100644
index 000000000..9c698626f
--- /dev/null
+++ b/youtube_dl/extractor/storyfire.py
@@ -0,0 +1,151 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import functools
+
+from .common import InfoExtractor
+from ..utils import (
+    # HEADRequest,
+    int_or_none,
+    OnDemandPagedList,
+    smuggle_url,
+)
+
+
+class StoryFireBaseIE(InfoExtractor):
+    _VALID_URL_BASE = r'https?://(?:www\.)?storyfire\.com/'
+
+    def _call_api(self, path, video_id, resource, query=None):
+        return self._download_json(
+            'https://storyfire.com/app/%s/%s' % (path, video_id), video_id,
+            'Downloading %s JSON metadata' % resource, query=query)
+
+    def _parse_video(self, video):
+        title = video['title']
+        vimeo_id = self._search_regex(
+            r'https?://player\.vimeo\.com/external/(\d+)',
+            video['vimeoVideoURL'], 'vimeo id')
+
+        # video_url = self._request_webpage(
+        #    HEADRequest(video['vimeoVideoURL']), video_id).geturl()
+        # formats = []
+        # for v_url, suffix in [(video_url, '_sep'), (video_url.replace('/sep/video/', '/video/'), '')]:
+        #    formats.extend(self._extract_m3u8_formats(
+        #        v_url, video_id, 'mp4', 'm3u8_native',
+        #        m3u8_id='hls' + suffix, fatal=False))
+        #    formats.extend(self._extract_mpd_formats(
+        #        v_url.replace('.m3u8', '.mpd'), video_id,
+        #        mpd_id='dash' + suffix, fatal=False))
+        # self._sort_formats(formats)
+
+        uploader_id = video.get('hostID')
+
+        return {
+            '_type': 'url_transparent',
+            'id': vimeo_id,
+            'title': title,
+            'description': video.get('description'),
+            'url': smuggle_url(
+                'https://player.vimeo.com/video/' + vimeo_id, {
+                    'http_headers': {
+                        'Referer': 'https://storyfire.com/',
+                    }
+                }),
+            # 'formats': formats,
+            'thumbnail': video.get('storyImage'),
+            'view_count': int_or_none(video.get('views')),
+            'like_count': int_or_none(video.get('likesCount')),
+            'comment_count': int_or_none(video.get('commentsCount')),
+            'duration': int_or_none(video.get('videoDuration')),
+            'timestamp': int_or_none(video.get('publishDate')),
+            'uploader': video.get('username'),
+            'uploader_id': uploader_id,
+            'uploader_url': 'https://storyfire.com/user/%s/video' % uploader_id if uploader_id else None,
+            'episode_number': int_or_none(video.get('episodeNumber') or video.get('episode_number')),
+        }
+
+
+class StoryFireIE(StoryFireBaseIE):
+    _VALID_URL = StoryFireBaseIE._VALID_URL_BASE + r'video-details/(?P<id>[0-9a-f]{24})'
+    _TEST = {
+        'url': 'https://storyfire.com/video-details/5df1d132b6378700117f9181',
+        'md5': 'caec54b9e4621186d6079c7ec100c1eb',
+        'info_dict': {
+            'id': '378954662',
+            'ext': 'mp4',
+            'title': 'Buzzfeed Teaches You About Memes',
+            'uploader_id': 'ntZAJFECERSgqHSxzonV5K2E89s1',
+            'timestamp': 1576129028,
+            'description': 'md5:0b4e28021548e144bed69bb7539e62ea',
+            'uploader': 'whang!',
+            'upload_date': '20191212',
+            'duration': 418,
+            'view_count': int,
+            'like_count': int,
+            'comment_count': int,
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'expected_warnings': ['Unable to download JSON metadata']
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        video = self._call_api(
+            'generic/video-detail', video_id, 'video')['video']
+        return self._parse_video(video)
+
+
+class StoryFireUserIE(StoryFireBaseIE):
+    _VALID_URL = StoryFireBaseIE._VALID_URL_BASE + r'user/(?P<id>[^/]+)/video'
+    _TEST = {
+        'url': 'https://storyfire.com/user/UQ986nFxmAWIgnkZQ0ftVhq4nOk2/video',
+        'info_dict': {
+            'id': 'UQ986nFxmAWIgnkZQ0ftVhq4nOk2',
+        },
+        'playlist_mincount': 151,
+    }
+    _PAGE_SIZE = 20
+
+    def _fetch_page(self, user_id, page):
+        videos = self._call_api(
+            'publicVideos', user_id, 'page %d' % (page + 1), {
+                'skip': page * self._PAGE_SIZE,
+            })['videos']
+        for video in videos:
+            yield self._parse_video(video)
+
+    def _real_extract(self, url):
+        user_id = self._match_id(url)
+        entries = OnDemandPagedList(functools.partial(
+            self._fetch_page, user_id), self._PAGE_SIZE)
+        return self.playlist_result(entries, user_id)
+
+
+class StoryFireSeriesIE(StoryFireBaseIE):
+    _VALID_URL = StoryFireBaseIE._VALID_URL_BASE + r'write/series/stories/(?P<id>[^/?&#]+)'
+    _TESTS = [{
+        'url': 'https://storyfire.com/write/series/stories/-Lq6MsuIHLODO6d2dDkr/',
+        'info_dict': {
+            'id': '-Lq6MsuIHLODO6d2dDkr',
+        },
+        'playlist_mincount': 13,
+    }, {
+        'url': 'https://storyfire.com/write/series/stories/the_mortal_one/',
+        'info_dict': {
+            'id': 'the_mortal_one',
+        },
+        'playlist_count': 0,
+    }]
+
+    def _extract_videos(self, stories):
+        for story in stories.values():
+            if story.get('hasVideo'):
+                yield self._parse_video(story)
+
+    def _real_extract(self, url):
+        series_id = self._match_id(url)
+        stories = self._call_api(
+            'seriesStories', series_id, 'series stories')
+        return self.playlist_result(self._extract_videos(stories), series_id)

From 646052e416577cc805b7ba169c49158716541570 Mon Sep 17 00:00:00 2001
From: Max <github@nbr23.com>
Date: Tue, 16 Feb 2021 15:22:51 -0500
Subject: [PATCH 215/860] [postprocessor/embedthumbnail] Recognize
 atomicparsley binary in lowercase (#28112)

---
 youtube_dl/postprocessor/embedthumbnail.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 5a3359588..3990908b6 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -89,10 +89,14 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
             os.rename(encodeFilename(temp_filename), encodeFilename(filename))
 
         elif info['ext'] in ['m4a', 'mp4']:
-            if not check_executable('AtomicParsley', ['-v']):
+            atomicparsley = next((x
+                                  for x in ['AtomicParsley', 'atomicparsley']
+                                  if check_executable(x, ['-v'])), None)
+
+            if atomicparsley is None:
                 raise EmbedThumbnailPPError('AtomicParsley was not found. Please install.')
 
-            cmd = [encodeFilename('AtomicParsley', True),
+            cmd = [encodeFilename(atomicparsley, True),
                    encodeFilename(filename, True),
                    encodeArgument('--artwork'),
                    encodeFilename(thumbnail_filename, True),

From a363fb5d28da7c1b651e6de98b9e799544a4df73 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 17 Feb 2021 04:03:54 +0700
Subject: [PATCH 216/860] [yandexmusic:playlist] Request missing tracks in
 chunks (closes #27355, closes #28184)

---
 youtube_dl/extractor/yandexmusic.py | 35 +++++++++++++++++++----------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 7893f363e..84969f8e1 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -1,8 +1,9 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
 import hashlib
+import itertools
+import re
 
 from .common import InfoExtractor
 from ..compat import compat_str
@@ -209,17 +210,27 @@ class YandexMusicPlaylistBaseIE(YandexMusicBaseIE):
             missing_track_ids = [
                 track_id for track_id in track_ids
                 if track_id not in present_track_ids]
-            missing_tracks = self._call_api(
-                'track-entries', tld, url, item_id,
-                'Downloading missing tracks JSON', {
-                    'entries': ','.join(missing_track_ids),
-                    'lang': tld,
-                    'external-domain': 'music.yandex.%s' % tld,
-                    'overembed': 'false',
-                    'strict': 'true',
-                })
-            if missing_tracks:
-                tracks.extend(missing_tracks)
+            # Request missing tracks in chunks to avoid exceeding max HTTP header size,
+            # see https://github.com/ytdl-org/youtube-dl/issues/27355
+            _TRACKS_PER_CHUNK = 250
+            for chunk_num in itertools.count(0):
+                start = chunk_num * _TRACKS_PER_CHUNK
+                end = start + _TRACKS_PER_CHUNK
+                missing_track_ids_req = missing_track_ids[start:end]
+                assert missing_track_ids_req
+                missing_tracks = self._call_api(
+                    'track-entries', tld, url, item_id,
+                    'Downloading missing tracks JSON chunk %d' % (chunk_num + 1), {
+                        'entries': ','.join(missing_track_ids_req),
+                        'lang': tld,
+                        'external-domain': 'music.yandex.%s' % tld,
+                        'overembed': 'false',
+                        'strict': 'true',
+                    })
+                if missing_tracks:
+                    tracks.extend(missing_tracks)
+                if end >= len(missing_track_ids):
+                    break
 
         return tracks
 

From 8980f53b4227bc213048fce52c634830dd25e4bb Mon Sep 17 00:00:00 2001
From: PrinceOfPuppers <joshuamcpherson5@gmail.com>
Date: Tue, 2 Feb 2021 01:46:39 -0500
Subject: [PATCH 217/860] [youtube] Fix uploader extraction in flat playlist
 mode (#28045)

---
 youtube_dl/extractor/youtube.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 7db4503e0..e0b15f859 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -308,7 +308,9 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             r'^([\d,]+)', re.sub(r'\s', '', view_count_text),
             'view count', default=None))
         uploader = try_get(
-            renderer, lambda x: x['ownerText']['runs'][0]['text'], compat_str)
+            renderer,
+            (lambda x: x['ownerText']['runs'][0]['text'],
+             lambda x: x['shortBylineText']['runs'][0]['text']), compat_str)
         return {
             '_type': 'url_transparent',
             'ie_key': YoutubeIE.ie_key(),

From 70baa7bfae8890c8274af7f3c7e2a704d300a326 Mon Sep 17 00:00:00 2001
From: PrinceOfPuppers <joshuamcpherson5@gmail.com>
Date: Mon, 15 Feb 2021 14:38:41 -0500
Subject: [PATCH 218/860] [test_youtube_lists] Actualize youtube flat playlist
 test (closes #28045)

---
 test/test_youtube_lists.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index c4f0abbea..cf2fdf14f 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -12,6 +12,7 @@ from test.helper import FakeYDL
 
 from youtube_dl.extractor import (
     YoutubePlaylistIE,
+    YoutubeTabIE,
     YoutubeIE,
 )
 
@@ -57,14 +58,22 @@ class TestYoutubeLists(unittest.TestCase):
         entries = result['entries']
         self.assertEqual(len(entries), 100)
 
-    def test_youtube_flat_playlist_titles(self):
+    def test_youtube_flat_playlist_extraction(self):
         dl = FakeYDL()
         dl.params['extract_flat'] = True
-        ie = YoutubePlaylistIE(dl)
-        result = ie.extract('https://www.youtube.com/playlist?list=PL-KKIb8rvtMSrAO9YFbeM6UQrAqoFTUWv')
+        ie = YoutubeTabIE(dl)
+        result = ie.extract('https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc')
         self.assertIsPlaylist(result)
-        for entry in result['entries']:
-            self.assertTrue(entry.get('title'))
+        entries = list(result['entries'])
+        self.assertTrue(len(entries) == 1)
+        video = entries[0]
+        self.assertEqual(video['_type'], 'url_transparent')
+        self.assertEqual(video['ie_key'], 'Youtube')
+        self.assertEqual(video['id'], 'BaW_jenozKc')
+        self.assertEqual(video['url'], 'BaW_jenozKc')
+        self.assertEqual(video['title'], 'youtube-dl test video "\'/\\ä↭𝕐')
+        self.assertEqual(video['duration'], 10)
+        self.assertEqual(video['uploader'], 'Philipp Hagemeister')
 
 
 if __name__ == '__main__':

From e20ec43094c09c41d71cef512c882a9d66163cd2 Mon Sep 17 00:00:00 2001
From: dmsummers <dsummers@cheesycactus.com>
Date: Thu, 20 Feb 2020 14:33:05 -0600
Subject: [PATCH 219/860] [simplecast] Add new extractor(closes #24107)

---
 youtube_dl/extractor/extractors.py |   5 +
 youtube_dl/extractor/generic.py    |  16 +++
 youtube_dl/extractor/simplecast.py | 160 +++++++++++++++++++++++++++++
 3 files changed, 181 insertions(+)
 create mode 100644 youtube_dl/extractor/simplecast.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 51f6d38e9..60c032c7d 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1057,6 +1057,11 @@ from .shared import (
     VivoIE,
 )
 from .showroomlive import ShowRoomLiveIE
+from .simplecast import (
+    SimplecastIE,
+    SimplecastEpisodeIE,
+    SimplecastPodcastIE,
+)
 from .sina import SinaIE
 from .sixplay import SixPlayIE
 from .skyit import (
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 09e680c96..c2b1b3bdf 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -129,6 +129,7 @@ from .odnoklassniki import OdnoklassnikiIE
 from .kinja import KinjaEmbedIE
 from .arcpublishing import ArcPublishingIE
 from .medialaan import MedialaanIE
+from .simplecast import SimplecastIE
 
 
 class GenericIE(InfoExtractor):
@@ -2238,6 +2239,15 @@ class GenericIE(InfoExtractor):
                 'duration': 159,
             },
         },
+        {
+            # Simplecast player embed
+            'url': 'https://www.bio.org/podcast',
+            'info_dict': {
+                'id': 'podcast',
+                'title': 'I AM BIO Podcast | BIO',
+            },
+            'playlist_mincount': 52,
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -2792,6 +2802,12 @@ class GenericIE(InfoExtractor):
             return self.playlist_from_matches(
                 matches, video_id, video_title, getter=unescapeHTML, ie='FunnyOrDie')
 
+        # Look for Simplecast embeds
+        simplecast_urls = SimplecastIE._extract_urls(webpage)
+        if simplecast_urls:
+            return self.playlist_from_matches(
+                simplecast_urls, video_id, video_title)
+
         # Look for BBC iPlayer embed
         matches = re.findall(r'setPlaylist\("(https?://www\.bbc\.co\.uk/iplayer/[^/]+/[\da-z]{8})"\)', webpage)
         if matches:
diff --git a/youtube_dl/extractor/simplecast.py b/youtube_dl/extractor/simplecast.py
new file mode 100644
index 000000000..2d0b3c06d
--- /dev/null
+++ b/youtube_dl/extractor/simplecast.py
@@ -0,0 +1,160 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_podcast_url,
+    int_or_none,
+    parse_iso8601,
+    strip_or_none,
+    try_get,
+    urlencode_postdata,
+)
+
+
+class SimplecastBaseIE(InfoExtractor):
+    _UUID_REGEX = r'[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12}'
+    _API_BASE = 'https://api.simplecast.com/'
+
+    def _call_api(self, path_tmpl, video_id):
+        return self._download_json(
+            self._API_BASE + path_tmpl % video_id, video_id)
+
+    def _call_search_api(self, resource, resource_id, resource_url):
+        return self._download_json(
+            'https://api.simplecast.com/%ss/search' % resource, resource_id,
+            data=urlencode_postdata({'url': resource_url}))
+
+    def _parse_episode(self, episode):
+        episode_id = episode['id']
+        title = episode['title'].strip()
+        audio_file = episode.get('audio_file') or {}
+        audio_file_url = audio_file.get('url') or episode.get('audio_file_url') or episode['enclosure_url']
+
+        season = episode.get('season') or {}
+        season_href = season.get('href')
+        season_id = None
+        if season_href:
+            season_id = self._search_regex(
+                r'https?://api.simplecast.com/seasons/(%s)' % self._UUID_REGEX,
+                season_href, 'season id', default=None)
+
+        webpage_url = episode.get('episode_url')
+        channel_url = None
+        if webpage_url:
+            channel_url = self._search_regex(
+                r'(https?://[^/]+\.simplecast\.com)',
+                webpage_url, 'channel url', default=None)
+
+        return {
+            'id': episode_id,
+            'display_id': episode.get('slug'),
+            'title': title,
+            'url': clean_podcast_url(audio_file_url),
+            'webpage_url': webpage_url,
+            'channel_url': channel_url,
+            'series': try_get(episode, lambda x: x['podcast']['title']),
+            'season_number': int_or_none(season.get('number')),
+            'season_id': season_id,
+            'thumbnail': episode.get('image_url'),
+            'episode_id': episode_id,
+            'episode_number': int_or_none(episode.get('number')),
+            'description': strip_or_none(episode.get('description')),
+            'timestamp': parse_iso8601(episode.get('published_at')),
+            'duration': int_or_none(episode.get('duration')),
+            'filesize': int_or_none(audio_file.get('size') or episode.get('audio_file_size')),
+        }
+
+
+class SimplecastIE(SimplecastBaseIE):
+    IE_NAME = 'simplecast'
+    _VALID_URL = r'https?://(?:api\.simplecast\.com/episodes|player\.simplecast\.com)/(?P<id>%s)' % SimplecastBaseIE._UUID_REGEX
+    _COMMON_TEST_INFO = {
+        'display_id': 'errant-signal-chris-franklin-new-wave-video-essays',
+        'id': 'b6dc49a2-9404-4853-9aa9-9cfc097be876',
+        'ext': 'mp3',
+        'title': 'Errant Signal - Chris Franklin & New Wave Video Essays',
+        'episode_number': 1,
+        'episode_id': 'b6dc49a2-9404-4853-9aa9-9cfc097be876',
+        'description': 'md5:34752789d3d2702e2d2c975fbd14f357',
+        'season_number': 1,
+        'season_id': 'e23df0da-bae4-4531-8bbf-71364a88dc13',
+        'series': 'The RE:BIND.io Podcast',
+        'duration': 5343,
+        'timestamp': 1580979475,
+        'upload_date': '20200206',
+        'webpage_url': r're:^https?://the-re-bind-io-podcast\.simplecast\.com/episodes/errant-signal-chris-franklin-new-wave-video-essays',
+        'channel_url': r're:^https?://the-re-bind-io-podcast\.simplecast\.com$',
+    }
+    _TESTS = [{
+        'url': 'https://api.simplecast.com/episodes/b6dc49a2-9404-4853-9aa9-9cfc097be876',
+        'md5': '8c93be7be54251bf29ee97464eabe61c',
+        'info_dict': _COMMON_TEST_INFO,
+    }, {
+        'url': 'https://player.simplecast.com/b6dc49a2-9404-4853-9aa9-9cfc097be876',
+        'only_matching': True,
+    }]
+
+    @staticmethod
+    def _extract_urls(webpage):
+        return re.findall(
+            r'''(?x)<iframe[^>]+src=["\']
+                (
+                    https?://(?:embed\.simplecast\.com/[0-9a-f]{8}|
+                    player\.simplecast\.com/%s
+                ))''' % SimplecastBaseIE._UUID_REGEX, webpage)
+
+    def _real_extract(self, url):
+        episode_id = self._match_id(url)
+        episode = self._call_api('episodes/%s', episode_id)
+        return self._parse_episode(episode)
+
+
+class SimplecastEpisodeIE(SimplecastBaseIE):
+    IE_NAME = 'simplecast:episode'
+    _VALID_URL = r'https?://(?!api\.)[^/]+\.simplecast\.com/episodes/(?P<id>[^/?&#]+)'
+    _TEST = {
+        'url': 'https://the-re-bind-io-podcast.simplecast.com/episodes/errant-signal-chris-franklin-new-wave-video-essays',
+        'md5': '8c93be7be54251bf29ee97464eabe61c',
+        'info_dict': SimplecastIE._COMMON_TEST_INFO,
+    }
+
+    def _real_extract(self, url):
+        mobj = re.match(self._VALID_URL, url)
+        episode = self._call_search_api(
+            'episode', mobj.group(1), mobj.group(0))
+        return self._parse_episode(episode)
+
+
+class SimplecastPodcastIE(SimplecastBaseIE):
+    IE_NAME = 'simplecast:podcast'
+    _VALID_URL = r'https?://(?!(?:api|cdn|embed|feeds|player)\.)(?P<id>[^/]+)\.simplecast\.com(?!/episodes/[^/?&#]+)'
+    _TESTS = [{
+        'url': 'https://the-re-bind-io-podcast.simplecast.com',
+        'playlist_mincount': 33,
+        'info_dict': {
+            'id': '07d28d26-7522-42eb-8c53-2bdcfc81c43c',
+            'title': 'The RE:BIND.io Podcast',
+        },
+    }, {
+        'url': 'https://the-re-bind-io-podcast.simplecast.com/episodes',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        subdomain = self._match_id(url)
+        site = self._call_search_api('site', subdomain, url)
+        podcast = site['podcast']
+        podcast_id = podcast['id']
+        podcast_title = podcast.get('title')
+
+        def entries():
+            episodes = self._call_api('podcasts/%s/episodes', podcast_id)
+            for episode in (episodes.get('collection') or []):
+                info = self._parse_episode(episode)
+                info['series'] = podcast_title
+                yield info
+
+        return self.playlist_result(entries(), podcast_id, podcast_title)

From a7356dffe90ed68958d839da073f1321f87a4feb Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 17 Feb 2021 18:33:33 +0100
Subject: [PATCH 220/860] [dplay] Add support for discoveryplus.com (closes
 #24698)

---
 youtube_dl/extractor/dplay.py      | 123 ++++++++++++++++++++++-------
 youtube_dl/extractor/extractors.py |   5 +-
 2 files changed, 99 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index 47501dbe6..540505719 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import json
 import re
 
 from .common import InfoExtractor
@@ -151,56 +152,79 @@ class DPlayIE(InfoExtractor):
         'only_matching': True,
     }]
 
+    def _process_errors(self, e, geo_countries):
+        info = self._parse_json(e.cause.read().decode('utf-8'), None)
+        error = info['errors'][0]
+        error_code = error.get('code')
+        if error_code == 'access.denied.geoblocked':
+            self.raise_geo_restricted(countries=geo_countries)
+        elif error_code in ('access.denied.missingpackage', 'invalid.token'):
+            raise ExtractorError(
+                'This video is only available for registered users. You may want to use --cookies.', expected=True)
+        raise ExtractorError(info['errors'][0]['detail'], expected=True)
+
+    def _update_disco_api_headers(self, headers, disco_base, display_id, realm):
+        headers['Authorization'] = 'Bearer ' + self._download_json(
+            disco_base + 'token', display_id, 'Downloading token',
+            query={
+                'realm': realm,
+            })['data']['attributes']['token']
+
+    def _download_video_playback_info(self, disco_base, video_id, headers):
+        streaming = self._download_json(
+            disco_base + 'playback/videoPlaybackInfo/' + video_id,
+            video_id, headers=headers)['data']['attributes']['streaming']
+        streaming_list = []
+        for format_id, format_dict in streaming.items():
+            streaming_list.append({
+                'type': format_id,
+                'url': format_dict.get('url'),
+            })
+        return streaming_list
+
     def _get_disco_api_info(self, url, display_id, disco_host, realm, country):
         geo_countries = [country.upper()]
         self._initialize_geo_bypass({
             'countries': geo_countries,
         })
         disco_base = 'https://%s/' % disco_host
-        token = self._download_json(
-            disco_base + 'token', display_id, 'Downloading token',
-            query={
-                'realm': realm,
-            })['data']['attributes']['token']
         headers = {
             'Referer': url,
-            'Authorization': 'Bearer ' + token,
         }
-        video = self._download_json(
-            disco_base + 'content/videos/' + display_id, display_id,
-            headers=headers, query={
-                'fields[channel]': 'name',
-                'fields[image]': 'height,src,width',
-                'fields[show]': 'name',
-                'fields[tag]': 'name',
-                'fields[video]': 'description,episodeNumber,name,publishStart,seasonNumber,videoDuration',
-                'include': 'images,primaryChannel,show,tags'
-            })
+        self._update_disco_api_headers(headers, disco_base, display_id, realm)
+        try:
+            video = self._download_json(
+                disco_base + 'content/videos/' + display_id, display_id,
+                headers=headers, query={
+                    'fields[channel]': 'name',
+                    'fields[image]': 'height,src,width',
+                    'fields[show]': 'name',
+                    'fields[tag]': 'name',
+                    'fields[video]': 'description,episodeNumber,name,publishStart,seasonNumber,videoDuration',
+                    'include': 'images,primaryChannel,show,tags'
+                })
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+                self._process_errors(e, geo_countries)
+            raise
         video_id = video['data']['id']
         info = video['data']['attributes']
         title = info['name'].strip()
         formats = []
         try:
-            streaming = self._download_json(
-                disco_base + 'playback/videoPlaybackInfo/' + video_id,
-                display_id, headers=headers)['data']['attributes']['streaming']
+            streaming = self._download_video_playback_info(
+                disco_base, video_id, headers)
         except ExtractorError as e:
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 403:
-                info = self._parse_json(e.cause.read().decode('utf-8'), display_id)
-                error = info['errors'][0]
-                error_code = error.get('code')
-                if error_code == 'access.denied.geoblocked':
-                    self.raise_geo_restricted(countries=geo_countries)
-                elif error_code == 'access.denied.missingpackage':
-                    self.raise_login_required()
-                raise ExtractorError(info['errors'][0]['detail'], expected=True)
+                self._process_errors(e, geo_countries)
             raise
-        for format_id, format_dict in streaming.items():
+        for format_dict in streaming:
             if not isinstance(format_dict, dict):
                 continue
             format_url = format_dict.get('url')
             if not format_url:
                 continue
+            format_id = format_dict.get('type')
             ext = determine_ext(format_url)
             if format_id == 'dash' or ext == 'mpd':
                 formats.extend(self._extract_mpd_formats(
@@ -268,3 +292,46 @@ class DPlayIE(InfoExtractor):
         host = 'disco-api.' + domain if domain[0] == 'd' else 'eu2-prod.disco-api.com'
         return self._get_disco_api_info(
             url, display_id, host, 'dplay' + country, country)
+
+
+class DiscoveryPlusIE(DPlayIE):
+    _VALID_URL = r'https?://(?:www\.)?discoveryplus\.com/video/(?P<id>[^/]+/[^/]+)'
+    _TESTS = [{
+        'url': 'https://www.discoveryplus.com/video/property-brothers-forever-home/food-and-family',
+        'info_dict': {
+            'id': '1140794',
+            'display_id': 'property-brothers-forever-home/food-and-family',
+            'ext': 'mp4',
+            'title': 'Food and Family',
+            'description': 'The brothers help a Richmond family expand their single-level home.',
+            'duration': 2583.113,
+            'timestamp': 1609304400,
+            'upload_date': '20201230',
+            'creator': 'HGTV',
+            'series': 'Property Brothers: Forever Home',
+            'season_number': 1,
+            'episode_number': 1,
+        },
+        'skip': 'Available for Premium users',
+    }]
+
+    def _update_disco_api_headers(self, headers, disco_base, display_id, realm):
+        headers['x-disco-client'] = 'WEB:UNKNOWN:dplus_us:15.0.0'
+
+    def _download_video_playback_info(self, disco_base, video_id, headers):
+        return self._download_json(
+            disco_base + 'playback/v3/videoPlaybackInfo',
+            video_id, headers=headers, data=json.dumps({
+                'deviceInfo': {
+                    'adBlocker': False,
+                },
+                'videoId': video_id,
+                'wisteriaProperties': {
+                    'platform': 'desktop',
+                },
+            }).encode('utf-8'))['data']['attributes']['streaming']
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        return self._get_disco_api_info(
+            url, display_id, 'us1-prod-direct.discoveryplus.com', 'go', 'us')
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 60c032c7d..acf8cf73b 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -288,7 +288,10 @@ from .douyutv import (
     DouyuShowIE,
     DouyuTVIE,
 )
-from .dplay import DPlayIE
+from .dplay import (
+    DPlayIE,
+    DiscoveryPlusIE,
+)
 from .dreisat import DreiSatIE
 from .drbonanza import DRBonanzaIE
 from .drtuber import DrTuberIE

From 3997efb65ef16dbd8c4792e79e797cbcab0fbec1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 17 Feb 2021 19:50:04 +0100
Subject: [PATCH 221/860] [dplay] add support for de.hgtv.com (closes #28182)

---
 youtube_dl/extractor/dplay.py      | 37 +++++++++++++++++++++++++++---
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 35 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index 540505719..0f0632f26 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -11,11 +11,13 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     int_or_none,
+    strip_or_none,
     unified_timestamp,
 )
 
 
 class DPlayIE(InfoExtractor):
+    _PATH_REGEX = r'/(?P<id>[^/]+/[^/?#]+)'
     _VALID_URL = r'''(?x)https?://
         (?P<domain>
             (?:www\.)?(?P<host>d
@@ -25,7 +27,7 @@ class DPlayIE(InfoExtractor):
                 )
             )|
             (?P<subdomain_country>es|it)\.dplay\.com
-        )/[^/]+/(?P<id>[^/]+/[^/?#]+)'''
+        )/[^/]+''' + _PATH_REGEX
 
     _TESTS = [{
         # non geo restricted, via secure api, unsigned download hls URL
@@ -272,7 +274,7 @@ class DPlayIE(InfoExtractor):
             'id': video_id,
             'display_id': display_id,
             'title': title,
-            'description': info.get('description'),
+            'description': strip_or_none(info.get('description')),
             'duration': float_or_none(info.get('videoDuration'), 1000),
             'timestamp': unified_timestamp(info.get('publishStart')),
             'series': series,
@@ -295,7 +297,7 @@ class DPlayIE(InfoExtractor):
 
 
 class DiscoveryPlusIE(DPlayIE):
-    _VALID_URL = r'https?://(?:www\.)?discoveryplus\.com/video/(?P<id>[^/]+/[^/]+)'
+    _VALID_URL = r'https?://(?:www\.)?discoveryplus\.com/video' + DPlayIE._PATH_REGEX
     _TESTS = [{
         'url': 'https://www.discoveryplus.com/video/property-brothers-forever-home/food-and-family',
         'info_dict': {
@@ -335,3 +337,32 @@ class DiscoveryPlusIE(DPlayIE):
         display_id = self._match_id(url)
         return self._get_disco_api_info(
             url, display_id, 'us1-prod-direct.discoveryplus.com', 'go', 'us')
+
+
+class HGTVDeIE(DPlayIE):
+    _VALID_URL = r'https?://de\.hgtv\.com/sendungen' + DPlayIE._PATH_REGEX
+    _TESTS = [{
+        'url': 'https://de.hgtv.com/sendungen/tiny-house-klein-aber-oho/wer-braucht-schon-eine-toilette/',
+        'info_dict': {
+            'id': '151205',
+            'display_id': 'tiny-house-klein-aber-oho/wer-braucht-schon-eine-toilette',
+            'ext': 'mp4',
+            'title': 'Wer braucht schon eine Toilette',
+            'description': 'md5:05b40a27e7aed2c9172de34d459134e2',
+            'duration': 1177.024,
+            'timestamp': 1595705400,
+            'upload_date': '20200725',
+            'creator': 'HGTV',
+            'series': 'Tiny House - klein, aber oho',
+            'season_number': 3,
+            'episode_number': 3,
+        },
+        'params': {
+            'format': 'bestvideo',
+        },
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        return self._get_disco_api_info(
+            url, display_id, 'eu1-prod.disco-api.com', 'hgtv', 'de')
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index acf8cf73b..62819ddcf 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -291,6 +291,7 @@ from .douyutv import (
 from .dplay import (
     DPlayIE,
     DiscoveryPlusIE,
+    HGTVDeIE,
 )
 from .dreisat import DreiSatIE
 from .drbonanza import DRBonanzaIE

From 08c2fbb8443de3949af205d08015e5c6048d2e86 Mon Sep 17 00:00:00 2001
From: bopol <bopol@e.email>
Date: Wed, 17 Feb 2021 22:29:32 +0100
Subject: [PATCH 222/860] [youtube] Add support for redirect.invidious.io
 (#28193)

Co-authored-by: Sergey M <dstftw@gmail.com>
---
 youtube_dl/extractor/youtube.py | 13 ++++++++++++-
 1 file changed, 12 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e0b15f859..f9e554ca9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -335,8 +335,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             (?:www\.)?hooktube\.com/|
                             (?:www\.)?yourepeat\.com/|
                             tube\.majestyc\.net/|
-                            # Invidious instances taken from https://github.com/omarroth/invidious/wiki/Invidious-Instances
+                            # invidious-redirect websites
+                            (?:www\.)?redirect\.invidious\.io/|
                             (?:(?:www|dev)\.)?invidio\.us/|
+                            # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/Invidious-Instances.md
                             (?:(?:www|no)\.)?invidiou\.sh/|
                             (?:(?:www|fi)\.)?invidious\.snopyta\.org/|
                             (?:www\.)?invidious\.kabi\.tk/|
@@ -906,6 +908,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'url': 'https://invidio.us/watch?v=BaW_jenozKc',
             'only_matching': True,
         },
+        {
+            'url': 'https://redirect.invidious.io/watch?v=BaW_jenozKc',
+            'only_matching': True,
+        },
+        {
+            # from https://nitter.pussthecat.org/YouTube/status/1360363141947944964#m
+            'url': 'https://redirect.invidious.io/Yh0AhrY9GjA',
+            'only_matching': True,
+        },
         {
             # DRM protected
             'url': 'https://www.youtube.com/watch?v=s7_qI6_mIXc',

From 9fc5eafb8e384453a49f7cfe73147be491f0b19d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 18 Feb 2021 04:59:56 +0700
Subject: [PATCH 223/860] [youtube] Improve _VALID_URL (refs #28193)

---
 youtube_dl/extractor/youtube.py | 99 ++++++++++++++++++---------------
 1 file changed, 53 insertions(+), 46 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f9e554ca9..ff32758df 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -326,54 +326,57 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
 class YoutubeIE(YoutubeBaseInfoExtractor):
     IE_DESC = 'YouTube.com'
+    _INVIDIOUS_SITES = (
+        # invidious-redirect websites
+        r'(?:www\.)?redirect\.invidious\.io',
+        r'(?:(?:www|dev)\.)?invidio\.us',
+        # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/Invidious-Instances.md
+        r'(?:(?:www|no)\.)?invidiou\.sh',
+        r'(?:(?:www|fi)\.)?invidious\.snopyta\.org',
+        r'(?:www\.)?invidious\.kabi\.tk',
+        r'(?:www\.)?invidious\.13ad\.de',
+        r'(?:www\.)?invidious\.mastodon\.host',
+        r'(?:www\.)?invidious\.zapashcanon\.fr',
+        r'(?:www\.)?invidious\.kavin\.rocks',
+        r'(?:www\.)?invidious\.tube',
+        r'(?:www\.)?invidiou\.site',
+        r'(?:www\.)?invidious\.site',
+        r'(?:www\.)?invidious\.xyz',
+        r'(?:www\.)?invidious\.nixnet\.xyz',
+        r'(?:www\.)?invidious\.drycat\.fr',
+        r'(?:www\.)?tube\.poal\.co',
+        r'(?:www\.)?tube\.connect\.cafe',
+        r'(?:www\.)?vid\.wxzm\.sx',
+        r'(?:www\.)?vid\.mint\.lgbt',
+        r'(?:www\.)?yewtu\.be',
+        r'(?:www\.)?yt\.elukerio\.org',
+        r'(?:www\.)?yt\.lelux\.fi',
+        r'(?:www\.)?invidious\.ggc-project\.de',
+        r'(?:www\.)?yt\.maisputain\.ovh',
+        r'(?:www\.)?invidious\.13ad\.de',
+        r'(?:www\.)?invidious\.toot\.koeln',
+        r'(?:www\.)?invidious\.fdn\.fr',
+        r'(?:www\.)?watch\.nettohikari\.com',
+        r'(?:www\.)?kgg2m7yk5aybusll\.onion',
+        r'(?:www\.)?qklhadlycap4cnod\.onion',
+        r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
+        r'(?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion',
+        r'(?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion',
+        r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
+        r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
+        r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
+    )
     _VALID_URL = r"""(?x)^
                      (
                          (?:https?://|//)                                    # http(s):// or protocol-independent URL
-                         (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com/|
-                            (?:www\.)?deturl\.com/www\.youtube\.com/|
-                            (?:www\.)?pwnyoutube\.com/|
-                            (?:www\.)?hooktube\.com/|
-                            (?:www\.)?yourepeat\.com/|
-                            tube\.majestyc\.net/|
-                            # invidious-redirect websites
-                            (?:www\.)?redirect\.invidious\.io/|
-                            (?:(?:www|dev)\.)?invidio\.us/|
-                            # Invidious instances taken from https://github.com/iv-org/documentation/blob/master/Invidious-Instances.md
-                            (?:(?:www|no)\.)?invidiou\.sh/|
-                            (?:(?:www|fi)\.)?invidious\.snopyta\.org/|
-                            (?:www\.)?invidious\.kabi\.tk/|
-                            (?:www\.)?invidious\.13ad\.de/|
-                            (?:www\.)?invidious\.mastodon\.host/|
-                            (?:www\.)?invidious\.zapashcanon\.fr/|
-                            (?:www\.)?invidious\.kavin\.rocks/|
-                            (?:www\.)?invidious\.tube/|
-                            (?:www\.)?invidiou\.site/|
-                            (?:www\.)?invidious\.site/|
-                            (?:www\.)?invidious\.xyz/|
-                            (?:www\.)?invidious\.nixnet\.xyz/|
-                            (?:www\.)?invidious\.drycat\.fr/|
-                            (?:www\.)?tube\.poal\.co/|
-                            (?:www\.)?tube\.connect\.cafe/|
-                            (?:www\.)?vid\.wxzm\.sx/|
-                            (?:www\.)?vid\.mint\.lgbt/|
-                            (?:www\.)?yewtu\.be/|
-                            (?:www\.)?yt\.elukerio\.org/|
-                            (?:www\.)?yt\.lelux\.fi/|
-                            (?:www\.)?invidious\.ggc-project\.de/|
-                            (?:www\.)?yt\.maisputain\.ovh/|
-                            (?:www\.)?invidious\.13ad\.de/|
-                            (?:www\.)?invidious\.toot\.koeln/|
-                            (?:www\.)?invidious\.fdn\.fr/|
-                            (?:www\.)?watch\.nettohikari\.com/|
-                            (?:www\.)?kgg2m7yk5aybusll\.onion/|
-                            (?:www\.)?qklhadlycap4cnod\.onion/|
-                            (?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion/|
-                            (?:www\.)?c7hqkpkpemu6e7emz5b4vyz7idjgdvgaaa3dyimmeojqbgpea3xqjoid\.onion/|
-                            (?:www\.)?fz253lmuao3strwbfbmx46yu7acac2jz27iwtorgmbqlkurlclmancad\.onion/|
-                            (?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion/|
-                            (?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p/|
-                            (?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion/|
-                            youtube\.googleapis\.com/)                        # the various hostnames, with wildcard subdomains
+                         (?:(?:(?:(?:\w+\.)?[yY][oO][uU][tT][uU][bB][eE](?:-nocookie|kids)?\.com|
+                            (?:www\.)?deturl\.com/www\.youtube\.com|
+                            (?:www\.)?pwnyoutube\.com|
+                            (?:www\.)?hooktube\.com|
+                            (?:www\.)?yourepeat\.com|
+                            tube\.majestyc\.net|
+                            %(invidious)s|
+                            youtube\.googleapis\.com)/                        # the various hostnames, with wildcard subdomains
                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
                          (?:                                                  # the various things that can precede the ID:
                              (?:(?:v|embed|e)/(?!videoseries))                # v/ or embed/ or e/
@@ -388,6 +391,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             youtu\.be|                                        # just youtu.be/xxxx
                             vid\.plus|                                        # or vid.plus/xxxx
                             zwearz\.com/watch|                                # or zwearz.com/watch/xxxx
+                            %(invidious)s
                          )/
                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
                          )
@@ -400,7 +404,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         )
                      )
                      (?(1).+)?                                                # if we found the ID, everything can follow
-                     $""" % {'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE}
+                     $""" % {
+        'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
+        'invidious': '|'.join(_INVIDIOUS_SITES),
+    }
     _PLAYER_INFO_RE = (
         r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',

From 40edffae3d9f86ca696dda6c8a4c9c0497cb6d76 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 19 Feb 2021 11:55:14 +0100
Subject: [PATCH 224/860] [ninegag] unscape title(#28201)

---
 youtube_dl/extractor/ninegag.py | 13 +++++++++----
 1 file changed, 9 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ninegag.py b/youtube_dl/extractor/ninegag.py
index 440f865bc..14390823b 100644
--- a/youtube_dl/extractor/ninegag.py
+++ b/youtube_dl/extractor/ninegag.py
@@ -2,10 +2,11 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    determine_ext,
     ExtractorError,
+    determine_ext,
     int_or_none,
     try_get,
+    unescapeHTML,
     url_or_none,
 )
 
@@ -14,7 +15,7 @@ class NineGagIE(InfoExtractor):
     IE_NAME = '9gag'
     _VALID_URL = r'https?://(?:www\.)?9gag\.com/gag/(?P<id>[^/?&#]+)'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'https://9gag.com/gag/ae5Ag7B',
         'info_dict': {
             'id': 'ae5Ag7B',
@@ -29,7 +30,11 @@ class NineGagIE(InfoExtractor):
             'dislike_count': int,
             'comment_count': int,
         }
-    }
+    }, {
+        # HTML escaped title
+        'url': 'https://9gag.com/gag/av5nvyb',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         post_id = self._match_id(url)
@@ -43,7 +48,7 @@ class NineGagIE(InfoExtractor):
                 'The given url does not contain a video',
                 expected=True)
 
-        title = post['title']
+        title = unescapeHTML(post['title'])
 
         duration = None
         formats = []

From b92bb0e02a09930cad3c4f6a406eb503c941af61 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 19 Feb 2021 16:00:22 +0100
Subject: [PATCH 225/860] [viki] improve extraction(closes #26522)(closes
 #28203)

- extract uploader_url and episode_number
- report login required error
- extract 480p formats
- fix API v4 calls
---
 youtube_dl/extractor/viki.py | 69 +++++++++++++++++++++++-------------
 1 file changed, 44 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index a311f21ef..2e9cbf148 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -21,6 +21,7 @@ from ..utils import (
     parse_iso8601,
     sanitized_Request,
     std_headers,
+    try_get,
 )
 
 
@@ -30,7 +31,7 @@ class VikiBaseIE(InfoExtractor):
     _API_URL_TEMPLATE = 'https://api.viki.io%s&sig=%s'
 
     _APP = '100005a'
-    _APP_VERSION = '2.2.5.1428709186'
+    _APP_VERSION = '6.0.0'
     _APP_SECRET = 'MM_d*yP@`&1@]@!AVrXf_o-HVEnoTnm$O-ti4[G~$JDI/Dc-&piU&z&5.;:}95=Iad'
 
     _GEO_BYPASS = False
@@ -41,7 +42,7 @@ class VikiBaseIE(InfoExtractor):
     _ERRORS = {
         'geo': 'Sorry, this content is not available in your region.',
         'upcoming': 'Sorry, this content is not yet available.',
-        # 'paywall': 'paywall',
+        'paywall': 'Sorry, this content is only available to Viki Pass Plus subscribers',
     }
 
     def _prepare_call(self, path, timestamp=None, post_data=None):
@@ -62,7 +63,8 @@ class VikiBaseIE(InfoExtractor):
 
     def _call_api(self, path, video_id, note, timestamp=None, post_data=None):
         resp = self._download_json(
-            self._prepare_call(path, timestamp, post_data), video_id, note)
+            self._prepare_call(path, timestamp, post_data), video_id, note,
+            headers={'x-viki-app-ver': self._APP_VERSION})
 
         error = resp.get('error')
         if error:
@@ -82,11 +84,13 @@ class VikiBaseIE(InfoExtractor):
             expected=True)
 
     def _check_errors(self, data):
-        for reason, status in data.get('blocking', {}).items():
+        for reason, status in (data.get('blocking') or {}).items():
             if status and reason in self._ERRORS:
                 message = self._ERRORS[reason]
                 if reason == 'geo':
                     self.raise_geo_restricted(msg=message)
+                elif reason == 'paywall':
+                    self.raise_login_required(message)
                 raise ExtractorError('%s said: %s' % (
                     self.IE_NAME, message), expected=True)
 
@@ -131,13 +135,19 @@ class VikiIE(VikiBaseIE):
         'info_dict': {
             'id': '1023585v',
             'ext': 'mp4',
-            'title': 'Heirs Episode 14',
-            'uploader': 'SBS',
-            'description': 'md5:c4b17b9626dd4b143dcc4d855ba3474e',
+            'title': 'Heirs - Episode 14',
+            'uploader': 'SBS Contents Hub',
+            'timestamp': 1385047627,
             'upload_date': '20131121',
             'age_limit': 13,
+            'duration': 3570,
+            'episode_number': 14,
+        },
+        'params': {
+            'format': 'bestvideo',
         },
         'skip': 'Blocked in the US',
+        'expected_warnings': ['Unknown MIME type image/jpeg in DASH manifest'],
     }, {
         # clip
         'url': 'http://www.viki.com/videos/1067139v-the-avengers-age-of-ultron-press-conference',
@@ -153,7 +163,8 @@ class VikiIE(VikiBaseIE):
             'uploader': 'Arirang TV',
             'like_count': int,
             'age_limit': 0,
-        }
+        },
+        'skip': 'Sorry. There was an error loading this video',
     }, {
         'url': 'http://www.viki.com/videos/1048879v-ankhon-dekhi',
         'info_dict': {
@@ -171,7 +182,7 @@ class VikiIE(VikiBaseIE):
     }, {
         # episode
         'url': 'http://www.viki.com/videos/44699v-boys-over-flowers-episode-1',
-        'md5': '94e0e34fd58f169f40c184f232356cfe',
+        'md5': '0a53dc252e6e690feccd756861495a8c',
         'info_dict': {
             'id': '44699v',
             'ext': 'mp4',
@@ -183,6 +194,10 @@ class VikiIE(VikiBaseIE):
             'uploader': 'group8',
             'like_count': int,
             'age_limit': 13,
+            'episode_number': 1,
+        },
+        'params': {
+            'format': 'bestvideo',
         },
         'expected_warnings': ['Unknown MIME type image/jpeg in DASH manifest'],
     }, {
@@ -209,7 +224,7 @@ class VikiIE(VikiBaseIE):
     }, {
         # non-English description
         'url': 'http://www.viki.com/videos/158036v-love-in-magic',
-        'md5': 'adf9e321a0ae5d0aace349efaaff7691',
+        'md5': '41faaba0de90483fb4848952af7c7d0d',
         'info_dict': {
             'id': '158036v',
             'ext': 'mp4',
@@ -220,6 +235,10 @@ class VikiIE(VikiBaseIE):
             'title': 'Love In Magic',
             'age_limit': 13,
         },
+        'params': {
+            'format': 'bestvideo',
+        },
+        'expected_warnings': ['Unknown MIME type image/jpeg in DASH manifest'],
     }]
 
     def _real_extract(self, url):
@@ -229,36 +248,33 @@ class VikiIE(VikiBaseIE):
             'https://www.viki.com/api/videos/' + video_id,
             video_id, 'Downloading video JSON', headers={
                 'x-client-user-agent': std_headers['User-Agent'],
-                'x-viki-app-ver': '4.0.57',
+                'x-viki-app-ver': '3.0.0',
             })
         video = resp['video']
 
         self._check_errors(video)
 
         title = self.dict_selection(video.get('titles', {}), 'en', allow_fallback=False)
+        episode_number = int_or_none(video.get('number'))
         if not title:
-            title = 'Episode %d' % video.get('number') if video.get('type') == 'episode' else video.get('id') or video_id
-            container_titles = video.get('container', {}).get('titles', {})
+            title = 'Episode %d' % episode_number if video.get('type') == 'episode' else video.get('id') or video_id
+            container_titles = try_get(video, lambda x: x['container']['titles'], dict) or {}
             container_title = self.dict_selection(container_titles, 'en')
             title = '%s - %s' % (container_title, title)
 
         description = self.dict_selection(video.get('descriptions', {}), 'en')
 
-        duration = int_or_none(video.get('duration'))
-        timestamp = parse_iso8601(video.get('created_at'))
-        uploader = video.get('author')
-        like_count = int_or_none(video.get('likes', {}).get('count'))
-        age_limit = parse_age_limit(video.get('rating'))
+        like_count = int_or_none(try_get(video, lambda x: x['likes']['count']))
 
         thumbnails = []
-        for thumbnail_id, thumbnail in video.get('images', {}).items():
+        for thumbnail_id, thumbnail in (video.get('images') or {}).items():
             thumbnails.append({
                 'id': thumbnail_id,
                 'url': thumbnail.get('url'),
             })
 
         subtitles = {}
-        for subtitle_lang, _ in video.get('subtitle_completions', {}).items():
+        for subtitle_lang, _ in (video.get('subtitle_completions') or {}).items():
             subtitles[subtitle_lang] = [{
                 'ext': subtitles_format,
                 'url': self._prepare_call(
@@ -269,13 +285,15 @@ class VikiIE(VikiBaseIE):
             'id': video_id,
             'title': title,
             'description': description,
-            'duration': duration,
-            'timestamp': timestamp,
-            'uploader': uploader,
+            'duration': int_or_none(video.get('duration')),
+            'timestamp': parse_iso8601(video.get('created_at')),
+            'uploader': video.get('author'),
+            'uploader_url': video.get('author_url'),
             'like_count': like_count,
-            'age_limit': age_limit,
+            'age_limit': parse_age_limit(video.get('rating')),
             'thumbnails': thumbnails,
             'subtitles': subtitles,
+            'episode_number': episode_number,
         }
 
         formats = []
@@ -360,7 +378,7 @@ class VikiChannelIE(VikiBaseIE):
         'info_dict': {
             'id': '50c',
             'title': 'Boys Over Flowers',
-            'description': 'md5:ecd3cff47967fe193cff37c0bec52790',
+            'description': 'md5:804ce6e7837e1fd527ad2f25420f4d59',
         },
         'playlist_mincount': 71,
     }, {
@@ -371,6 +389,7 @@ class VikiChannelIE(VikiBaseIE):
             'description': 'md5:05bf5471385aa8b21c18ad450e350525',
         },
         'playlist_count': 127,
+        'skip': 'Page not found',
     }, {
         'url': 'http://www.viki.com/news/24569c-showbiz-korea',
         'only_matching': True,

From cf2dbec6301177a1fddf72862de05fa912d9869d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 19 Feb 2021 21:13:56 +0100
Subject: [PATCH 226/860] [vimeo] add support for unlisted video source format
 extraction

---
 youtube_dl/extractor/vimeo.py | 18 +++++++++++++-----
 1 file changed, 13 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 15cd06268..bd2663fe0 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -226,10 +226,12 @@ class VimeoBaseInfoExtractor(InfoExtractor):
             'is_live': is_live,
         }
 
-    def _extract_original_format(self, url, video_id):
+    def _extract_original_format(self, url, video_id, unlisted_hash=None):
+        query = {'action': 'load_download_config'}
+        if unlisted_hash:
+            query['unlisted_hash'] = unlisted_hash
         download_data = self._download_json(
-            url, video_id, fatal=False,
-            query={'action': 'load_download_config'},
+            url, video_id, fatal=False, query=query,
             headers={'X-Requested-With': 'XMLHttpRequest'})
         if download_data:
             source_file = download_data.get('source_file')
@@ -509,6 +511,11 @@ class VimeoIE(VimeoBaseInfoExtractor):
         {
             'url': 'https://vimeo.com/160743502/abd0e13fb4',
             'only_matching': True,
+        },
+        {
+            # requires passing unlisted_hash(a52724358e) to load_download_config request
+            'url': 'https://vimeo.com/392479337/a52724358e',
+            'only_matching': True,
         }
         # https://gettingthingsdone.com/workflowmap/
         # vimeo embed with check-password page protected by Referer header
@@ -673,7 +680,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
             if config.get('view') == 4:
                 config = self._verify_player_video_password(redirect_url, video_id, headers)
 
-        vod = config.get('video', {}).get('vod', {})
+        video = config.get('video') or {}
+        vod = video.get('vod') or {}
 
         def is_rented():
             if '>You rented this title.<' in webpage:
@@ -733,7 +741,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         formats = []
 
         source_format = self._extract_original_format(
-            'https://vimeo.com/' + video_id, video_id)
+            'https://vimeo.com/' + video_id, video_id, video.get('unlisted_hash'))
         if source_format:
             formats.append(source_format)
 

From 21e872b19ada61337770160a124c4387d6c77e08 Mon Sep 17 00:00:00 2001
From: Isaac-the-Man <steven97102@gmail.com>
Date: Sun, 10 Jan 2021 10:37:54 -0500
Subject: [PATCH 227/860] [samplefocus] Add new extractor(closes #27763)

---
 youtube_dl/extractor/extractors.py  |   1 +
 youtube_dl/extractor/samplefocus.py | 100 ++++++++++++++++++++++++++++
 2 files changed, 101 insertions(+)
 create mode 100644 youtube_dl/extractor/samplefocus.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 62819ddcf..1a39c25c5 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1029,6 +1029,7 @@ from .safari import (
     SafariApiIE,
     SafariCourseIE,
 )
+from .samplefocus import SampleFocusIE
 from .sapo import SapoIE
 from .savefrom import SaveFromIE
 from .sbs import SBSIE
diff --git a/youtube_dl/extractor/samplefocus.py b/youtube_dl/extractor/samplefocus.py
new file mode 100644
index 000000000..806c3c354
--- /dev/null
+++ b/youtube_dl/extractor/samplefocus.py
@@ -0,0 +1,100 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    extract_attributes,
+    get_element_by_attribute,
+    int_or_none,
+)
+
+
+class SampleFocusIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?samplefocus\.com/samples/(?P<id>[^/?&#]+)'
+    _TESTS = [{
+        'url': 'https://samplefocus.com/samples/lil-peep-sad-emo-guitar',
+        'md5': '48c8d62d60be467293912e0e619a5120',
+        'info_dict': {
+            'id': '40316',
+            'display_id': 'lil-peep-sad-emo-guitar',
+            'ext': 'mp3',
+            'title': 'Lil Peep Sad Emo Guitar',
+            'thumbnail': r're:^https?://.+\.png',
+            'license': 'Standard License',
+            'uploader': 'CapsCtrl',
+            'uploader_id': 'capsctrl',
+            'like_count': int,
+            'comment_count': int,
+            'categories': ['Samples', 'Guitar', 'Electric guitar'],
+        },
+    }, {
+        'url': 'https://samplefocus.com/samples/dababy-style-bass-808',
+        'only_matching': True
+    }, {
+        'url': 'https://samplefocus.com/samples/young-chop-kick',
+        'only_matching': True
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+
+        sample_id = self._search_regex(
+            r'<input[^>]+id=(["\'])sample_id\1[^>]+value=(?:["\'])(?P<id>\d+)',
+            webpage, 'sample id', group='id')
+
+        title = self._og_search_title(webpage, fatal=False) or self._html_search_regex(
+            r'<h1>(.+?)</h1>', webpage, 'title')
+
+        mp3_url = self._search_regex(
+            r'<input[^>]+id=(["\'])sample_mp3\1[^>]+value=(["\'])(?P<url>(?:(?!\2).)+)',
+            webpage, 'mp3', fatal=False, group='url') or extract_attributes(self._search_regex(
+                r'<meta[^>]+itemprop=(["\'])contentUrl\1[^>]*>',
+                webpage, 'mp3 url', group=0))['content']
+
+        thumbnail = self._og_search_thumbnail(webpage) or self._html_search_regex(
+            r'<img[^>]+class=(?:["\'])waveform responsive-img[^>]+src=(["\'])(?P<url>(?:(?!\1).)+)',
+            webpage, 'mp3', fatal=False, group='url')
+
+        comments = []
+        for author_id, author, body in re.findall(r'(?s)<p[^>]+class="comment-author"><a[^>]+href="/users/([^"]+)">([^"]+)</a>.+?<p[^>]+class="comment-body">([^>]+)</p>', webpage):
+            comments.append({
+                'author': author,
+                'author_id': author_id,
+                'text': body,
+            })
+
+        uploader_id = uploader = None
+        mobj = re.search(r'>By <a[^>]+href="/users/([^"]+)"[^>]*>([^<]+)', webpage)
+        if mobj:
+            uploader_id, uploader = mobj.groups()
+
+        breadcrumb = get_element_by_attribute('typeof', 'BreadcrumbList', webpage)
+        categories = []
+        if breadcrumb:
+            for _, name in re.findall(r'<span[^>]+property=(["\'])name\1[^>]*>([^<]+)', breadcrumb):
+                categories.append(name)
+
+        def extract_count(klass):
+            return int_or_none(self._html_search_regex(
+                r'<span[^>]+class=(?:["\'])?%s-count[^>]*>(\d+)' % klass,
+                webpage, klass, fatal=False))
+
+        return {
+            'id': sample_id,
+            'title': title,
+            'url': mp3_url,
+            'display_id': display_id,
+            'thumbnail': thumbnail,
+            'uploader': uploader,
+            'license': self._html_search_regex(
+                r'<a[^>]+href=(["\'])/license\1[^>]*>(?P<license>[^<]+)<',
+                webpage, 'license', fatal=False, group='license'),
+            'uploader_id': uploader_id,
+            'like_count': extract_count('sample-%s-favorites' % sample_id),
+            'comment_count': extract_count('comments'),
+            'comments': comments,
+            'categories': categories,
+        }

From 3037ab00c7ddbe4bedaff51420e4ea1e8d0ccccb Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 21 Feb 2021 10:24:58 +0100
Subject: [PATCH 228/860] [youtube] fixup m4a_dash formats(closes #28165)

---
 youtube_dl/extractor/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ff32758df..72d9fbbc6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1603,6 +1603,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     # Youtube throttles chunks >~10M
                     'http_chunk_size': 10485760,
                 }
+                if dct.get('ext'):
+                    dct['container'] = dct['ext'] + '_dash'
             formats.append(dct)
 
         hls_manifest_url = streaming_data.get('hlsManifestUrl')

From f90d825a6be852b6a3fa39b0948cc9b94154963e Mon Sep 17 00:00:00 2001
From: SirCipherz <sircipherz@mail.com>
Date: Sun, 21 Feb 2021 16:05:33 +0000
Subject: [PATCH 229/860] [peertube] Add support for canard.tube (#28190)

---
 youtube_dl/extractor/peertube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/peertube.py b/youtube_dl/extractor/peertube.py
index c2ca71c71..32ff51653 100644
--- a/youtube_dl/extractor/peertube.py
+++ b/youtube_dl/extractor/peertube.py
@@ -413,7 +413,8 @@ class PeerTubeIE(InfoExtractor):
                             peertube3\.cpy\.re|
                             peertube2\.cpy\.re|
                             videos\.tcit\.fr|
-                            peertube\.cpy\.re
+                            peertube\.cpy\.re|
+                            canard\.tube
                         )'''
     _UUID_RE = r'[\da-fA-F]{8}-[\da-fA-F]{4}-[\da-fA-F]{4}-[\da-fA-F]{4}-[\da-fA-F]{12}'
     _API_BASE = 'https://%s/api/v1/videos/%s/%s'

From 696183e1333aa8f2f1241e149759edf410f94c79 Mon Sep 17 00:00:00 2001
From: piplongrun <piplongrun@users.noreply.github.com>
Date: Sun, 21 Feb 2021 17:19:37 +0100
Subject: [PATCH 230/860] [youporn] Extract duration (#28019)

Co-authored-by: Sergey M <dstftw@gmail.com>
---
 youtube_dl/extractor/youporn.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 534270bac..2b5771828 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -25,6 +25,7 @@ class YouPornIE(InfoExtractor):
             'title': 'Sex Ed: Is It Safe To Masturbate Daily?',
             'description': 'Love & Sex Answers: http://bit.ly/DanAndJenn -- Is It Unhealthy To Masturbate Daily?',
             'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 210,
             'uploader': 'Ask Dan And Jennifer',
             'upload_date': '20101217',
             'average_rating': int,
@@ -153,6 +154,8 @@ class YouPornIE(InfoExtractor):
         thumbnail = self._search_regex(
             r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
             webpage, 'thumbnail', fatal=False, group='thumbnail')
+        duration = int_or_none(self._html_search_meta(
+            'video:duration', webpage, 'duration', fatal=False))
 
         uploader = self._html_search_regex(
             r'(?s)<div[^>]+class=["\']submitByLink["\'][^>]*>(.+?)</div>',
@@ -194,6 +197,7 @@ class YouPornIE(InfoExtractor):
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
+            'duration': duration,
             'uploader': uploader,
             'upload_date': upload_date,
             'average_rating': average_rating,

From 919d7646004ad8480016b9dec0f6033759244520 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 21 Feb 2021 23:21:38 +0700
Subject: [PATCH 231/860] [youporn] Skip test

---
 youtube_dl/extractor/youporn.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 2b5771828..33114363d 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -55,6 +55,7 @@ class YouPornIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+        'skip': '404',
     }, {
         'url': 'https://www.youporn.com/embed/505835/sex-ed-is-it-safe-to-masturbate-daily/',
         'only_matching': True,

From 36abc16c3cf050a3368367038d40cce27504c28a Mon Sep 17 00:00:00 2001
From: Adrian Heine <mail@adrianheine.de>
Date: Sat, 9 Jan 2021 22:06:24 +0100
Subject: [PATCH 232/860] [apa] Fix extraction

---
 youtube_dl/extractor/apa.py | 27 ++++++++-------------------
 1 file changed, 8 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/apa.py b/youtube_dl/extractor/apa.py
index 98ccdaa4a..1dd35dd9c 100644
--- a/youtube_dl/extractor/apa.py
+++ b/youtube_dl/extractor/apa.py
@@ -7,7 +7,6 @@ from .common import InfoExtractor
 from ..utils import (
     determine_ext,
     js_to_json,
-    url_or_none,
 )
 
 
@@ -17,14 +16,10 @@ class APAIE(InfoExtractor):
         'url': 'http://uvp.apa.at/embed/293f6d17-692a-44e3-9fd5-7b178f3a1029',
         'md5': '2b12292faeb0a7d930c778c7a5b4759b',
         'info_dict': {
-            'id': 'jjv85FdZ',
+            'id': '293f6d17-692a-44e3-9fd5-7b178f3a1029',
             'ext': 'mp4',
-            'title': '"Blau ist mysteriös": Die Blue Man Group im Interview',
-            'description': 'md5:d41d8cd98f00b204e9800998ecf8427e',
+            'title': '293f6d17-692a-44e3-9fd5-7b178f3a1029',
             'thumbnail': r're:^https?://.*\.jpg$',
-            'duration': 254,
-            'timestamp': 1519211149,
-            'upload_date': '20180221',
         },
     }, {
         'url': 'https://uvp-apapublisher.sf.apa.at/embed/2f94e9e6-d945-4db2-9548-f9a41ebf7b78',
@@ -48,7 +43,7 @@ class APAIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        webpage = self._download_webpage('https://uvp.apa.at/player/%s' % video_id, video_id)
 
         jwplatform_id = self._search_regex(
             r'media[iI]d\s*:\s*["\'](?P<id>[a-zA-Z0-9]{8})', webpage,
@@ -59,18 +54,12 @@ class APAIE(InfoExtractor):
                 'jwplatform:' + jwplatform_id, ie='JWPlatform',
                 video_id=video_id)
 
-        sources = self._parse_json(
-            self._search_regex(
-                r'sources\s*=\s*(\[.+?\])\s*;', webpage, 'sources'),
-            video_id, transform_source=js_to_json)
+        sources = self._parse_json("{" + self._search_regex(
+            r'("hls"\s*:\s*"[^"]+"\s*,\s*"progressive"\s*:\s*"[^"]+")', webpage, 'sources')
+            + "}", video_id, transform_source=js_to_json)
 
         formats = []
-        for source in sources:
-            if not isinstance(source, dict):
-                continue
-            source_url = url_or_none(source.get('file'))
-            if not source_url:
-                continue
+        for (format, source_url) in sources.items():
             ext = determine_ext(source_url)
             if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
@@ -83,7 +72,7 @@ class APAIE(InfoExtractor):
         self._sort_formats(formats)
 
         thumbnail = self._search_regex(
-            r'image\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
+            r'"poster"\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
             'thumbnail', fatal=False, group='url')
 
         return {

From aa9118a373a6e9cfb9fda24533df86286eccc468 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Feb 2021 02:29:50 +0700
Subject: [PATCH 233/860] [apa] Improve extraction (closes #27750)

---
 youtube_dl/extractor/apa.py | 38 ++++++++++++++++++++++++-------------
 1 file changed, 25 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/apa.py b/youtube_dl/extractor/apa.py
index 1dd35dd9c..cbc1c0ecb 100644
--- a/youtube_dl/extractor/apa.py
+++ b/youtube_dl/extractor/apa.py
@@ -6,12 +6,13 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     determine_ext,
-    js_to_json,
+    int_or_none,
+    url_or_none,
 )
 
 
 class APAIE(InfoExtractor):
-    _VALID_URL = r'https?://[^/]+\.apa\.at/embed/(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
+    _VALID_URL = r'(?P<base_url>https?://[^/]+\.apa\.at)/embed/(?P<id>[\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'
     _TESTS = [{
         'url': 'http://uvp.apa.at/embed/293f6d17-692a-44e3-9fd5-7b178f3a1029',
         'md5': '2b12292faeb0a7d930c778c7a5b4759b',
@@ -41,9 +42,11 @@ class APAIE(InfoExtractor):
                 webpage)]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        mobj = re.match(self._VALID_URL, url)
+        video_id, base_url = mobj.group('id', 'base_url')
 
-        webpage = self._download_webpage('https://uvp.apa.at/player/%s' % video_id, video_id)
+        webpage = self._download_webpage(
+            '%s/player/%s' % (base_url, video_id), video_id)
 
         jwplatform_id = self._search_regex(
             r'media[iI]d\s*:\s*["\'](?P<id>[a-zA-Z0-9]{8})', webpage,
@@ -54,30 +57,39 @@ class APAIE(InfoExtractor):
                 'jwplatform:' + jwplatform_id, ie='JWPlatform',
                 video_id=video_id)
 
-        sources = self._parse_json("{" + self._search_regex(
-            r'("hls"\s*:\s*"[^"]+"\s*,\s*"progressive"\s*:\s*"[^"]+")', webpage, 'sources')
-            + "}", video_id, transform_source=js_to_json)
+        def extract(field, name=None):
+            return self._search_regex(
+                r'\b%s["\']\s*:\s*(["\'])(?P<value>(?:(?!\1).)+)\1' % field,
+                webpage, name or field, default=None, group='value')
+
+        title = extract('title') or video_id
+        description = extract('description')
+        thumbnail = extract('poster', 'thumbnail')
 
         formats = []
-        for (format, source_url) in sources.items():
+        for format_id in ('hls', 'progressive'):
+            source_url = url_or_none(extract(format_id))
+            if not source_url:
+                continue
             ext = determine_ext(source_url)
             if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
                     source_url, video_id, 'mp4', entry_protocol='m3u8_native',
                     m3u8_id='hls', fatal=False))
             else:
+                height = int_or_none(self._search_regex(
+                    r'(\d+)\.mp4', source_url, 'height', default=None))
                 formats.append({
                     'url': source_url,
+                    'format_id': format_id,
+                    'height': height,
                 })
         self._sort_formats(formats)
 
-        thumbnail = self._search_regex(
-            r'"poster"\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
-            'thumbnail', fatal=False, group='url')
-
         return {
             'id': video_id,
-            'title': video_id,
+            'title': title,
+            'description': description,
             'thumbnail': thumbnail,
             'formats': formats,
         }

From 44b2d5f5fc80a291b093c8bf20e2ad7ac58b3536 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Feb 2021 02:40:00 +0700
Subject: [PATCH 234/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 37 +++++++++++++++++++++++++++++++++++++
 1 file changed, 37 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 384bd19c2..69ce51890 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,40 @@
+version <unreleased>
+
+Core
++ [postprocessor/embedthumbnail] Recognize atomicparsley binary in lowercase
+  (#28112)
+
+Extractors
+* [apa] Fix and improve extraction (#27750)
++ [youporn] Extract duration (#28019)
++ [peertube] Add support for canard.tube (#28190)
+* [youtube] Fixup m4a_dash formats (#28165)
++ [samplefocus] Add support for samplefocus.com (#27763)
++ [vimeo] Add support for unlisted video source format extraction
+* [viki] Improve extraction (#26522, #28203)
+    * Extract uploader URL and episode number
+    * Report login required error
+    + Extract 480p formats
+    * Fix API v4 calls
+* [ninegag] Unescape title (#28201)
+* [youtube] Improve URL regular expression (#28193)
++ [youtube] Add support for redirect.invidious.io (#28193)
++ [dplay] Add support for de.hgtv.com (#28182)
++ [dplay] Add support for discoveryplus.com (#24698)
++ [simplecast] Add support for simplecast.com (#24107)
+* [youtube] Fix uploader extraction in flat playlist mode (#28045)
+* [yandexmusic:playlist] Request missing tracks in chunks (#27355, #28184)
++ [storyfire] Add support for storyfire.com (#25628, #26349)
++ [zhihu] Add support for zhihu.com (#28177)
+* [youtube] Fix controversial videos when authenticated with cookies (#28174)
+* [ccma] Fix timestamp parsing in python 2
++ [videopress] Add support for video.wordpress.com
+* [kakao] Improve info extraction and detect geo restriction (#26577)
+* [xboxclips] Fix extraction (#27151)
+* [ard] Improve formats extraction (#28155)
++ [canvas] Add support for dagelijksekost.een.be (#28119)
+
+
 version 2021.02.10
 
 Extractors

From 0a04e03a0245d78593844e7b7930920051b9cc27 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 22 Feb 2021 02:42:16 +0700
Subject: [PATCH 235/860] release 2021.02.22

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          |  6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md |  4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md |  4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           |  6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      |  4 ++--
 ChangeLog                                        |  2 +-
 docs/supportedsites.md                           | 11 +++++++++++
 youtube_dl/version.py                            |  2 +-
 8 files changed, 25 insertions(+), 14 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index ea0a59dca..60879f0ac 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.10. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.22. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.02.10**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.22**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.02.10
+ [debug] youtube-dl version 2021.02.22
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index d24855c72..b38d39ab4 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.10. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.22. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.02.10**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.22**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 8b96a2883..3235de44b 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.10. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.22. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.02.10**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.22**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index e46971047..a3255623a 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.10. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.22. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.02.10**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.22**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.02.10
+ [debug] youtube-dl version 2021.02.22
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index a9ca379ca..124b020c3 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.10. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.22. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.02.10**
+- [ ] I've verified that I'm running youtube-dl version **2021.02.22**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index 69ce51890..2912d776c 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.02.22
 
 Core
 + [postprocessor/embedthumbnail] Recognize atomicparsley binary in lowercase
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 1373cc4f6..2452c1f7f 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -212,6 +212,7 @@
  - **curiositystream**
  - **curiositystream:collection**
  - **CWTV**
+ - **DagelijkseKost**: dagelijksekost.een.be
  - **DailyMail**
  - **dailymotion**
  - **dailymotion:playlist**
@@ -233,6 +234,7 @@
  - **DiscoveryGo**
  - **DiscoveryGoPlaylist**
  - **DiscoveryNetworksDe**
+ - **DiscoveryPlus**
  - **DiscoveryVR**
  - **Disney**
  - **dlive:stream**
@@ -353,6 +355,7 @@
  - **HentaiStigma**
  - **hetklokhuis**
  - **hgtv.com:show**
+ - **HGTVDe**
  - **HiDive**
  - **HistoricFilms**
  - **history:player**
@@ -803,6 +806,7 @@
  - **safari:course**: safaribooksonline.com online courses
  - **SAKTV**
  - **SaltTV**
+ - **SampleFocus**
  - **Sapo**: SAPO Vídeos
  - **savefrom.net**
  - **SBS**: sbs.com.au
@@ -825,6 +829,9 @@
  - **ShahidShow**
  - **Shared**: shared.sx
  - **ShowRoomLive**
+ - **simplecast**
+ - **simplecast:episode**
+ - **simplecast:podcast**
  - **Sina**
  - **sky.it**
  - **sky:news**
@@ -877,6 +884,9 @@
  - **Steam**
  - **Stitcher**
  - **StitcherShow**
+ - **StoryFire**
+ - **StoryFireSeries**
+ - **StoryFireUser**
  - **Streamable**
  - **streamcloud.eu**
  - **StreamCZ**
@@ -1198,5 +1208,6 @@
  - **ZattooLive**
  - **ZDF**
  - **ZDFChannel**
+ - **Zhihu**
  - **zingmp3**: mp3.zing.vn
  - **Zype**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 79d2be625..f89530293 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.02.10'
+__version__ = '2021.02.22'

From 2090dbdc8c51d18760957e248f5ff152209f9236 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 21 Feb 2021 23:08:40 +0100
Subject: [PATCH 236/860] [youtube] fix get_video_info request

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 72d9fbbc6..2496d27f1 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1450,7 +1450,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'Refetching age-gated info webpage',
                     'unable to download video info webpage', query={
                         'video_id': video_id,
-                        'eurl': 'https://www.youtube.com/embed/' + video_id,
+                        'eurl': 'https://youtube.googleapis.com/v/' + video_id,
                     }, fatal=False)),
                 lambda x: x['player_response'][0],
                 compat_str) or '{}', video_id)

From 7422a2194fcbc179083c6927a2fcca278fed39c5 Mon Sep 17 00:00:00 2001
From: nixxo <c.nixxo@gmail.com>
Date: Thu, 15 Oct 2020 14:24:17 +0200
Subject: [PATCH 237/860] [gedidigital] Add new extractor(closes #7347)(closes
 #26946)

---
 youtube_dl/extractor/extractors.py  |   1 +
 youtube_dl/extractor/gedidigital.py | 161 ++++++++++++++++++++++++++++
 2 files changed, 162 insertions(+)
 create mode 100644 youtube_dl/extractor/gedidigital.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 1a39c25c5..dc6a06771 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -421,6 +421,7 @@ from .gamestar import GameStarIE
 from .gaskrank import GaskrankIE
 from .gazeta import GazetaIE
 from .gdcvault import GDCVaultIE
+from .gedidigital import GediDigitalIE
 from .generic import GenericIE
 from .gfycat import GfycatIE
 from .giantbomb import GiantBombIE
diff --git a/youtube_dl/extractor/gedidigital.py b/youtube_dl/extractor/gedidigital.py
new file mode 100644
index 000000000..1b47a4e27
--- /dev/null
+++ b/youtube_dl/extractor/gedidigital.py
@@ -0,0 +1,161 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    int_or_none,
+)
+
+
+class GediDigitalIE(InfoExtractor):
+    _VALID_URL = r'''(?x)https?://video\.
+        (?:
+            (?:
+                (?:espresso\.)?repubblica
+                |lastampa
+                |ilsecoloxix
+            )|
+            (?:
+                iltirreno
+                |messaggeroveneto
+                |ilpiccolo
+                |gazzettadimantova
+                |mattinopadova
+                |laprovinciapavese
+                |tribunatreviso
+                |nuovavenezia
+                |gazzettadimodena
+                |lanuovaferrara
+                |corrierealpi
+                |lasentinella
+            )\.gelocal
+        )\.it(?:/[^/]+){2,3}/(?P<id>\d+)'''
+    _TESTS = [{
+        'url': 'https://video.lastampa.it/politica/il-paradosso-delle-regionali-la-lega-vince-ma-sembra-aver-perso/121559/121683',
+        'md5': '84658d7fb9e55a6e57ecc77b73137494',
+        'info_dict': {
+            'id': '121559',
+            'ext': 'mp4',
+            'title': 'Il paradosso delle Regionali: ecco perché la Lega vince ma sembra aver perso',
+            'description': 'md5:de7f4d6eaaaf36c153b599b10f8ce7ca',
+            'thumbnail': r're:^https://www\.repstatic\.it/video/photo/.+?-thumb-full-.+?\.jpg$',
+            'duration': 125,
+        },
+    }, {
+        'url': 'https://video.espresso.repubblica.it/embed/tutti-i-video/01-ted-villa/14772/14870&width=640&height=360',
+        'only_matching': True,
+    }, {
+        'url': 'https://video.repubblica.it/motori/record-della-pista-a-spa-francorchamps-la-pagani-huayra-roadster-bc-stupisce/367415/367963',
+        'only_matching': True,
+    }, {
+        'url': 'https://video.ilsecoloxix.it/sport/cassani-e-i-brividi-azzurri-ai-mondiali-di-imola-qui-mi-sono-innamorato-del-ciclismo-da-ragazzino-incredibile-tornarci-da-ct/66184/66267',
+        'only_matching': True,
+    }, {
+        'url': 'https://video.iltirreno.gelocal.it/sport/dentro-la-notizia-ferrari-cosa-succede-a-maranello/141059/142723',
+        'only_matching': True,
+    }, {
+        'url': 'https://video.messaggeroveneto.gelocal.it/locale/maria-giovanna-elmi-covid-vaccino/138155/139268',
+        'only_matching': True,
+    }, {
+        'url': 'https://video.ilpiccolo.gelocal.it/dossier/big-john/dinosauro-big-john-al-via-le-visite-guidate-a-trieste/135226/135751',
+        'only_matching': True,
+    }, {
+        'url': 'https://video.gazzettadimantova.gelocal.it/locale/dal-ponte-visconteo-di-valeggio-l-and-8217sos-dei-ristoratori-aprire-anche-a-cena/137310/137818',
+        'only_matching': True,
+    }, {
+        'url': 'https://video.mattinopadova.gelocal.it/dossier/coronavirus-in-veneto/covid-a-vo-un-anno-dopo-un-cuore-tricolore-per-non-dimenticare/138402/138964',
+        'only_matching': True,
+    }, {
+        'url': 'https://video.laprovinciapavese.gelocal.it/locale/mede-zona-rossa-via-alle-vaccinazioni-per-gli-over-80/137545/138120',
+        'only_matching': True,
+    }, {
+        'url': 'https://video.tribunatreviso.gelocal.it/dossier/coronavirus-in-veneto/ecco-le-prima-vaccinazioni-di-massa-nella-marca/134485/135024',
+        'only_matching': True,
+    }, {
+        'url': 'https://video.nuovavenezia.gelocal.it/locale/camion-troppo-alto-per-il-ponte-ferroviario-perde-il-carico/135734/136266',
+        'only_matching': True,
+    }, {
+        'url': 'https://video.gazzettadimodena.gelocal.it/locale/modena-scoperta-la-proteina-che-predice-il-livello-di-gravita-del-covid/139109/139796',
+        'only_matching': True,
+    }, {
+        'url': 'https://video.lanuovaferrara.gelocal.it/locale/due-bombole-di-gpl-aperte-e-abbandonate-i-vigili-bruciano-il-gas/134391/134957',
+        'only_matching': True,
+    }, {
+        'url': 'https://video.corrierealpi.gelocal.it/dossier/cortina-2021-i-mondiali-di-sci-alpino/mondiali-di-sci-il-timelapse-sulla-splendida-olympia/133760/134331',
+        'only_matching': True,
+    }, {
+        'url': 'https://video.lasentinella.gelocal.it/locale/vestigne-centra-un-auto-e-si-ribalta/138931/139466',
+        'only_matching': True,
+    }, {
+        'url': 'https://video.espresso.repubblica.it/tutti-i-video/01-ted-villa/14772',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        title = self._html_search_meta(
+            ['twitter:title', 'og:title'], webpage, fatal=True)
+        player_data = re.findall(
+            r"PlayerFactory\.setParam\('(?P<type>format|param)',\s*'(?P<name>[^']+)',\s*'(?P<val>[^']+)'\);",
+            webpage)
+
+        formats = []
+        duration = thumb = None
+        for t, n, v in player_data:
+            if t == 'format':
+                if n in ('video-hds-vod-ec', 'video-hls-vod-ec', 'video-viralize', 'video-youtube-pfp'):
+                    continue
+                elif n.endswith('-vod-ak'):
+                    formats.extend(self._extract_akamai_formats(
+                        v, video_id, {'http': 'media.gedidigital.it'}))
+                else:
+                    ext = determine_ext(v)
+                    if ext == 'm3u8':
+                        formats.extend(self._extract_m3u8_formats(
+                            v, video_id, 'mp4', 'm3u8_native', m3u8_id=n, fatal=False))
+                        continue
+                    f = {
+                        'format_id': n,
+                        'url': v,
+                    }
+                    if ext == 'mp3':
+                        abr = int_or_none(self._search_regex(
+                            r'-mp3-audio-(\d+)', v, 'abr', default=None))
+                        f.update({
+                            'abr': abr,
+                            'tbr': abr,
+                            'vcodec': 'none'
+                        })
+                    else:
+                        mobj = re.match(r'^video-rrtv-(\d+)(?:-(\d+))?$', n)
+                        if mobj:
+                            f.update({
+                                'height': int(mobj.group(1)),
+                                'vbr': int_or_none(mobj.group(2)),
+                            })
+                        if not f.get('vbr'):
+                            f['vbr'] = int_or_none(self._search_regex(
+                                r'-video-rrtv-(\d+)', v, 'abr', default=None))
+                    formats.append(f)
+            elif t == 'param':
+                if n in ['image_full', 'image']:
+                    thumb = v
+                elif n == 'videoDuration':
+                    duration = int_or_none(v)
+
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': self._html_search_meta(
+                ['twitter:description', 'og:description', 'description'], webpage),
+            'thumbnail': thumb or self._og_search_thumbnail(webpage),
+            'formats': formats,
+            'duration': duration,
+        }

From d81421af4b4c3f8f6e197ad4a06fcdb948484c24 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 22 Feb 2021 23:02:15 +0100
Subject: [PATCH 238/860] [gedidigital] improve asset id matching

---
 youtube_dl/extractor/gedidigital.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gedidigital.py b/youtube_dl/extractor/gedidigital.py
index 1b47a4e27..6c4153b40 100644
--- a/youtube_dl/extractor/gedidigital.py
+++ b/youtube_dl/extractor/gedidigital.py
@@ -32,7 +32,7 @@ class GediDigitalIE(InfoExtractor):
                 |corrierealpi
                 |lasentinella
             )\.gelocal
-        )\.it(?:/[^/]+){2,3}/(?P<id>\d+)'''
+        )\.it(?:/[^/]+){2,3}?/(?P<id>\d+)(?:[/?&#]|$)'''
     _TESTS = [{
         'url': 'https://video.lastampa.it/politica/il-paradosso-delle-regionali-la-lega-vince-ma-sembra-aver-perso/121559/121683',
         'md5': '84658d7fb9e55a6e57ecc77b73137494',

From 8cb4b71909e720a758a17dd519d198e77884a14a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 23 Feb 2021 18:37:06 +0700
Subject: [PATCH 239/860] [tmz] Fix and improve extraction (closes #24603,
 closes #24687, closes #28211)

---
 youtube_dl/extractor/tmz.py | 101 ++++++++++++++++++++++++++++--------
 1 file changed, 78 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/tmz.py b/youtube_dl/extractor/tmz.py
index 419f9d92e..3d1bf75ff 100644
--- a/youtube_dl/extractor/tmz.py
+++ b/youtube_dl/extractor/tmz.py
@@ -2,55 +2,110 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
+from .jwplatform import JWPlatformIE
+from .kaltura import KalturaIE
+from ..utils import (
+    int_or_none,
+    unified_timestamp,
+)
 
 
 class TMZIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tmz\.com/videos/(?P<id>[^/?#]+)'
+    _VALID_URL = r'https?://(?:www\.)?tmz\.com/videos/(?P<id>[^/?#&]+)'
     _TESTS = [{
-        'url': 'http://www.tmz.com/videos/0_okj015ty/',
-        'md5': '4d22a51ef205b6c06395d8394f72d560',
-        'info_dict': {
-            'id': '0_okj015ty',
-            'ext': 'mp4',
-            'title': 'Kim Kardashian\'s Boobs Unlock a Mystery!',
-            'description': 'Did Kim Kardasain try to one-up Khloe by one-upping Kylie???  Or is she just showing off her amazing boobs?',
-            'timestamp': 1394747163,
-            'uploader_id': 'batchUser',
-            'upload_date': '20140313',
-        }
-    }, {
         'url': 'http://www.tmz.com/videos/0-cegprt2p/',
+        'md5': '31f9223e20eef55954973359afa61a20',
+        'info_dict': {
+            'id': 'P6YjLBLk',
+            'ext': 'mp4',
+            'title': "No Charges Against Hillary Clinton? Harvey Says It Ain't Over Yet",
+            'description': 'md5:b714359fc18607715ebccbd2da8ff488',
+            'timestamp': 1467831837,
+            'upload_date': '20160706',
+        },
+        'add_ie': [JWPlatformIE.ie_key()],
+    }, {
+        'url': 'http://www.tmz.com/videos/0_okj015ty/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.tmz.com/videos/071119-chris-morgan-women-4590005-0-zcsejvcr/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.tmz.com/videos/2021-02-19-021921-floyd-mayweather-1043872/',
         'only_matching': True,
     }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url).replace('-', '_')
-        return self.url_result('kaltura:591531:%s' % video_id, 'Kaltura', video_id)
+
+        webpage = self._download_webpage(url, video_id, fatal=False)
+        if webpage:
+            tmz_video_id = self._search_regex(
+                r'nodeRef\s*:\s*["\']tmz:video:([\da-fA-F]{8}-[\da-fA-F]{4}-[\da-fA-F]{4}-[\da-fA-F]{4}-[\da-fA-F]{12})',
+                webpage, 'video id', default=None)
+            video = self._download_json(
+                'https://www.tmz.com/_/video/%s' % tmz_video_id, video_id,
+                fatal=False)
+            if video:
+                message = video['message']
+                info = {
+                    '_type': 'url_transparent',
+                    'title': message.get('title'),
+                    'description': message.get('description'),
+                    'timestamp': unified_timestamp(message.get('published_at')),
+                    'duration': int_or_none(message.get('duration')),
+                }
+                jwplatform_id = message.get('jwplayer_media_id')
+                if jwplatform_id:
+                    info.update({
+                        'url': 'jwplatform:%s' % jwplatform_id,
+                        'ie_key': JWPlatformIE.ie_key(),
+                    })
+                else:
+                    kaltura_entry_id = message.get('kaltura_entry_id') or video_id
+                    kaltura_partner_id = message.get('kaltura_partner_id') or '591531'
+                    info.update({
+                        'url': 'kaltura:%s:%s' % (kaltura_partner_id, kaltura_entry_id),
+                        'ie_key': KalturaIE.ie_key(),
+                    })
+                return info
+
+        return self.url_result(
+            'kaltura:591531:%s' % video_id, KalturaIE.ie_key(), video_id)
 
 
 class TMZArticleIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?tmz\.com/\d{4}/\d{2}/\d{2}/(?P<id>[^/]+)/?'
+    _VALID_URL = r'https?://(?:www\.)?tmz\.com/\d{4}/\d{2}/\d{2}/(?P<id>[^/?#&]+)'
     _TEST = {
         'url': 'http://www.tmz.com/2015/04/19/bobby-brown-bobbi-kristina-awake-video-concert',
-        'md5': '3316ff838ae5bb7f642537825e1e90d2',
         'info_dict': {
-            'id': '0_6snoelag',
-            'ext': 'mov',
+            'id': 'PAKZa97W',
+            'ext': 'mp4',
             'title': 'Bobby Brown Tells Crowd ... Bobbi Kristina is Awake',
             'description': 'Bobby Brown stunned his audience during a concert Saturday night, when he told the crowd, "Bobbi is awake.  She\'s watching me."',
-            'timestamp': 1429467813,
+            'timestamp': 1429466400,
             'upload_date': '20150419',
-            'uploader_id': 'batchUser',
-        }
+        },
+        'params': {
+            'skip_download': True,
+        },
+        'add_ie': [JWPlatformIE.ie_key()],
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(url, video_id)
+
+        tmz_url = self._search_regex(
+            r'clickLink\s*\(\s*["\'](?P<url>%s)' % TMZIE._VALID_URL, webpage,
+            'video id', default=None, group='url')
+        if tmz_url:
+            return self.url_result(tmz_url, ie=TMZIE.ie_key())
+
         embedded_video_info = self._parse_json(self._html_search_regex(
             r'tmzVideoEmbed\(({.+?})\);', webpage, 'embedded video info'),
             video_id)
-
         return self.url_result(
-            'http://www.tmz.com/videos/%s/' % embedded_video_info['id'])
+            'http://www.tmz.com/videos/%s/' % embedded_video_info['id'],
+            ie=TMZIE.ie_key())

From 295860ff00c5d8caf94badd4f04671f6a631fcae Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 23 Feb 2021 12:39:46 +0100
Subject: [PATCH 240/860] [tf1] improve extraction(closes #27980)(closes
 #28040)

---
 youtube_dl/extractor/tf1.py | 127 +++++++++++++++++-------------------
 youtube_dl/extractor/wat.py |  95 ++++++++++++---------------
 2 files changed, 101 insertions(+), 121 deletions(-)

diff --git a/youtube_dl/extractor/tf1.py b/youtube_dl/extractor/tf1.py
index 55e2a0721..23c2808a1 100644
--- a/youtube_dl/extractor/tf1.py
+++ b/youtube_dl/extractor/tf1.py
@@ -1,92 +1,87 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import json
+import re
+
 from .common import InfoExtractor
-from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    parse_iso8601,
+    try_get,
+)
 
 
 class TF1IE(InfoExtractor):
-    """TF1 uses the wat.tv player."""
-    _VALID_URL = r'https?://(?:(?:videos|www|lci)\.tf1|(?:www\.)?(?:tfou|ushuaiatv|histoire|tvbreizh))\.fr/(?:[^/]+/)*(?P<id>[^/?#.]+)'
+    _VALID_URL = r'https?://(?:www\.)?tf1\.fr/[^/]+/(?P<program_slug>[^/]+)/videos/(?P<id>[^/?&#]+)\.html'
     _TESTS = [{
-        'url': 'http://videos.tf1.fr/auto-moto/citroen-grand-c4-picasso-2013-presentation-officielle-8062060.html',
-        'info_dict': {
-            'id': '10635995',
-            'ext': 'mp4',
-            'title': 'Citroën Grand C4 Picasso 2013 : présentation officielle',
-            'description': 'Vidéo officielle du nouveau Citroën Grand C4 Picasso, lancé à l\'automne 2013.',
-        },
-        'params': {
-            # Sometimes wat serves the whole file with the --test option
-            'skip_download': True,
-        },
-        'expected_warnings': ['HTTP Error 404'],
-    }, {
-        'url': 'http://www.tfou.fr/chuggington/videos/le-grand-mysterioso-chuggington-7085291-739.html',
-        'info_dict': {
-            'id': 'le-grand-mysterioso-chuggington-7085291-739',
-            'ext': 'mp4',
-            'title': 'Le grand Mystérioso - Chuggington',
-            'description': 'Le grand Mystérioso - Emery rêve qu\'un article lui soit consacré dans le journal.',
-            'upload_date': '20150103',
-        },
-        'params': {
-            # Sometimes wat serves the whole file with the --test option
-            'skip_download': True,
-        },
-        'skip': 'HTTP Error 410: Gone',
-    }, {
-        'url': 'http://www.tf1.fr/tf1/koh-lanta/videos/replay-koh-lanta-22-mai-2015.html',
-        'only_matching': True,
-    }, {
-        'url': 'http://lci.tf1.fr/sept-a-huit/videos/sept-a-huit-du-24-mai-2015-8611550.html',
-        'only_matching': True,
-    }, {
-        'url': 'http://www.tf1.fr/hd1/documentaire/videos/mylene-farmer-d-une-icone.html',
-        'only_matching': True,
-    }, {
         'url': 'https://www.tf1.fr/tmc/quotidien-avec-yann-barthes/videos/quotidien-premiere-partie-11-juin-2019.html',
         'info_dict': {
             'id': '13641379',
             'ext': 'mp4',
             'title': 'md5:f392bc52245dc5ad43771650c96fb620',
-            'description': 'md5:44bc54f0a21322f5b91d68e76a544eae',
+            'description': 'md5:a02cdb217141fb2d469d6216339b052f',
             'upload_date': '20190611',
+            'timestamp': 1560273989,
+            'duration': 1738,
+            'series': 'Quotidien avec Yann Barthès',
+            'tags': ['intégrale', 'quotidien', 'Replay'],
         },
         'params': {
             # Sometimes wat serves the whole file with the --test option
             'skip_download': True,
+            'format': 'bestvideo',
         },
+    }, {
+        'url': 'http://www.tf1.fr/tf1/koh-lanta/videos/replay-koh-lanta-22-mai-2015.html',
+        'only_matching': True,
+    }, {
+        'url': 'http://www.tf1.fr/hd1/documentaire/videos/mylene-farmer-d-une-icone.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
+        program_slug, slug = re.match(self._VALID_URL, url).groups()
+        video = self._download_json(
+            'https://www.tf1.fr/graphql/web', slug, query={
+                'id': '9b80783950b85247541dd1d851f9cc7fa36574af015621f853ab111a679ce26f',
+                'variables': json.dumps({
+                    'programSlug': program_slug,
+                    'slug': slug,
+                })
+            })['data']['videoBySlug']
+        wat_id = video['streamId']
 
-        webpage = self._download_webpage(url, video_id)
+        tags = []
+        for tag in (video.get('tags') or []):
+            label = tag.get('label')
+            if not label:
+                continue
+            tags.append(label)
 
-        wat_id = None
+        decoration = video.get('decoration') or {}
 
-        data = self._parse_json(
-            self._search_regex(
-                r'__APOLLO_STATE__\s*=\s*({.+?})\s*(?:;|</script>)', webpage,
-                'data', default='{}'), video_id, fatal=False)
+        thumbnails = []
+        for source in (try_get(decoration, lambda x: x['image']['sources'], list) or []):
+            source_url = source.get('url')
+            if not source_url:
+                continue
+            thumbnails.append({
+                'url': source_url,
+                'width': int_or_none(source.get('width')),
+            })
 
-        if data:
-            try:
-                wat_id = next(
-                    video.get('streamId')
-                    for key, video in data.items()
-                    if isinstance(video, dict)
-                    and video.get('slug') == video_id)
-                if not isinstance(wat_id, compat_str) or not wat_id.isdigit():
-                    wat_id = None
-            except StopIteration:
-                pass
-
-        if not wat_id:
-            wat_id = self._html_search_regex(
-                (r'(["\'])(?:https?:)?//www\.wat\.tv/embedframe/.*?(?P<id>\d{8})\1',
-                 r'(["\']?)streamId\1\s*:\s*(["\']?)(?P<id>\d+)\2'),
-                webpage, 'wat id', group='id')
-
-        return self.url_result('wat:%s' % wat_id, 'Wat')
+        return {
+            '_type': 'url_transparent',
+            'id': wat_id,
+            'url': 'wat:' + wat_id,
+            'title': video.get('title'),
+            'thumbnails': thumbnails,
+            'description': decoration.get('description'),
+            'timestamp': parse_iso8601(video.get('date')),
+            'duration': int_or_none(try_get(video, lambda x: x['publicPlayingInfos']['duration'])),
+            'tags': tags,
+            'series': decoration.get('programLabel'),
+            'season_number': int_or_none(video.get('season')),
+            'episode_number': int_or_none(video.get('episode')),
+        }
diff --git a/youtube_dl/extractor/wat.py b/youtube_dl/extractor/wat.py
index f6940b371..147931d73 100644
--- a/youtube_dl/extractor/wat.py
+++ b/youtube_dl/extractor/wat.py
@@ -4,9 +4,10 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
-    unified_strdate,
-    HEADRequest,
+    ExtractorError,
     int_or_none,
+    try_get,
+    unified_strdate,
 )
 
 
@@ -29,6 +30,7 @@ class WatIE(InfoExtractor):
                 'skip_download': True,
             },
             'expected_warnings': ['HTTP Error 404'],
+            'skip': 'This content is no longer available',
         },
         {
             'url': 'http://www.wat.tv/video/gregory-lemarchal-voix-ange-6z1v7_6ygkj_.html',
@@ -40,8 +42,10 @@ class WatIE(InfoExtractor):
                 'upload_date': '20140816',
             },
             'expected_warnings': ["Ce contenu n'est pas disponible pour l'instant."],
+            'skip': 'This content is no longer available',
         },
     ]
+    _GEO_BYPASS = False
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -49,71 +53,52 @@ class WatIE(InfoExtractor):
 
         # 'contentv4' is used in the website, but it also returns the related
         # videos, we don't need them
+        # video_data = self._download_json(
+        #     'http://www.wat.tv/interface/contentv4s/' + video_id, video_id)
         video_data = self._download_json(
-            'http://www.wat.tv/interface/contentv4s/' + video_id, video_id)
+            'https://mediainfo.tf1.fr/mediainfocombo/' + video_id,
+            video_id, query={'context': 'MYTF1'})
         video_info = video_data['media']
 
         error_desc = video_info.get('error_desc')
         if error_desc:
-            self.report_warning(
-                '%s returned error: %s' % (self.IE_NAME, error_desc))
+            if video_info.get('error_code') == 'GEOBLOCKED':
+                self.raise_geo_restricted(error_desc, video_info.get('geoList'))
+            raise ExtractorError(error_desc)
 
-        chapters = video_info['chapters']
-        if chapters:
-            first_chapter = chapters[0]
-
-            def video_id_for_chapter(chapter):
-                return chapter['tc_start'].split('-')[0]
-
-            if video_id_for_chapter(first_chapter) != video_id:
-                self.to_screen('Multipart video detected')
-                entries = [self.url_result('wat:%s' % video_id_for_chapter(chapter)) for chapter in chapters]
-                return self.playlist_result(entries, video_id, video_info['title'])
-            # Otherwise we can continue and extract just one part, we have to use
-            # the video id for getting the video url
-        else:
-            first_chapter = video_info
-
-        title = first_chapter['title']
-
-        def extract_url(path_template, url_type):
-            req_url = 'http://www.wat.tv/get/%s' % (path_template % video_id)
-            head = self._request_webpage(HEADRequest(req_url), video_id, 'Extracting %s url' % url_type, fatal=False)
-            if head:
-                red_url = head.geturl()
-                if req_url != red_url:
-                    return red_url
-            return None
+        title = video_info['title']
 
         formats = []
-        manifest_urls = self._download_json(
-            'http://www.wat.tv/get/webhtml/' + video_id, video_id)
-        m3u8_url = manifest_urls.get('hls')
-        if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4',
-                'm3u8_native', m3u8_id='hls', fatal=False))
-        mpd_url = manifest_urls.get('mpd')
-        if mpd_url:
-            formats.extend(self._extract_mpd_formats(
-                mpd_url.replace('://das-q1.tf1.fr/', '://das-q1-ssl.tf1.fr/'),
-                video_id, mpd_id='dash', fatal=False))
-        self._sort_formats(formats)
 
-        date_diffusion = first_chapter.get('date_diffusion') or video_data.get('configv4', {}).get('estatS4')
-        upload_date = unified_strdate(date_diffusion) if date_diffusion else None
-        duration = None
-        files = video_info['files']
-        if files:
-            duration = int_or_none(files[0].get('duration'))
+        def extract_formats(manifest_urls):
+            for f, f_url in manifest_urls.items():
+                if not f_url:
+                    continue
+                if f in ('dash', 'mpd'):
+                    formats.extend(self._extract_mpd_formats(
+                        f_url.replace('://das-q1.tf1.fr/', '://das-q1-ssl.tf1.fr/'),
+                        video_id, mpd_id='dash', fatal=False))
+                elif f == 'hls':
+                    formats.extend(self._extract_m3u8_formats(
+                        f_url, video_id, 'mp4',
+                        'm3u8_native', m3u8_id='hls', fatal=False))
+
+        delivery = video_data.get('delivery') or {}
+        extract_formats({delivery.get('format'): delivery.get('url')})
+        if not formats:
+            manifest_urls = self._download_json(
+                'http://www.wat.tv/get/webhtml/' + video_id, video_id, fatal=False)
+            if manifest_urls:
+                extract_formats(manifest_urls)
+
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': title,
-            'thumbnail': first_chapter.get('preview'),
-            'description': first_chapter.get('description'),
-            'view_count': int_or_none(video_info.get('views')),
-            'upload_date': upload_date,
-            'duration': duration,
+            'thumbnail': video_info.get('preview'),
+            'upload_date': unified_strdate(try_get(
+                video_data, lambda x: x['mediametrie']['chapters'][0]['estatS4'])),
+            'duration': int_or_none(video_info.get('duration')),
             'formats': formats,
         }

From 1631fca1ee1c3312027c702854d741bbb8025dcd Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 23 Feb 2021 13:50:18 +0100
Subject: [PATCH 241/860] [wat] detect DRM protected videos(closes #27958)

---
 youtube_dl/extractor/wat.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wat.py b/youtube_dl/extractor/wat.py
index 147931d73..f1bccc2d6 100644
--- a/youtube_dl/extractor/wat.py
+++ b/youtube_dl/extractor/wat.py
@@ -64,7 +64,7 @@ class WatIE(InfoExtractor):
         if error_desc:
             if video_info.get('error_code') == 'GEOBLOCKED':
                 self.raise_geo_restricted(error_desc, video_info.get('geoList'))
-            raise ExtractorError(error_desc)
+            raise ExtractorError(error_desc, expected=True)
 
         title = video_info['title']
 
@@ -86,6 +86,8 @@ class WatIE(InfoExtractor):
         delivery = video_data.get('delivery') or {}
         extract_formats({delivery.get('format'): delivery.get('url')})
         if not formats:
+            if delivery.get('drm'):
+                raise ExtractorError('This video is DRM protected.', expected=True)
             manifest_urls = self._download_json(
                 'http://www.wat.tv/get/webhtml/' + video_id, video_id, fatal=False)
             if manifest_urls:

From 44603290e5002153f3ebad6230cc73aef42cc2cd Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 24 Feb 2021 18:34:28 +0100
Subject: [PATCH 242/860] [dplay] Extract Ad-Free uplynk URLs(#28160)

---
 youtube_dl/extractor/dplay.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/dplay.py b/youtube_dl/extractor/dplay.py
index 0f0632f26..bbb199094 100644
--- a/youtube_dl/extractor/dplay.py
+++ b/youtube_dl/extractor/dplay.py
@@ -330,6 +330,7 @@ class DiscoveryPlusIE(DPlayIE):
                 'videoId': video_id,
                 'wisteriaProperties': {
                     'platform': 'desktop',
+                    'product': 'dplus_us',
                 },
             }).encode('utf-8'))['data']['attributes']['streaming']
 

From 9662e4964b8d1b8d23c79f90d91b9be87d10029f Mon Sep 17 00:00:00 2001
From: nixxo <nixxo@protonmail.com>
Date: Wed, 24 Feb 2021 22:17:29 +0100
Subject: [PATCH 243/860] [vvvvid] extract series sublists playlist_title
 (#27601) (#27618)

---
 youtube_dl/extractor/vvvvid.py | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vvvvid.py b/youtube_dl/extractor/vvvvid.py
index 778ce8b76..d62404cf3 100644
--- a/youtube_dl/extractor/vvvvid.py
+++ b/youtube_dl/extractor/vvvvid.py
@@ -247,9 +247,13 @@ class VVVVIDShowIE(VVVVIDIE):
         show_info = self._download_info(
             show_id, 'info/', show_title, fatal=False)
 
+        if not show_title:
+            base_url += "/title"
+
         entries = []
         for season in (seasons or []):
             episodes = season.get('episodes') or []
+            playlist_title = season.get('name') or show_info.get('title')
             for episode in episodes:
                 if episode.get('playable') is False:
                     continue
@@ -259,12 +263,13 @@ class VVVVIDShowIE(VVVVIDIE):
                     continue
                 info = self._extract_common_video_info(episode)
                 info.update({
-                    '_type': 'url',
+                    '_type': 'url_transparent',
                     'ie_key': VVVVIDIE.ie_key(),
                     'url': '/'.join([base_url, season_id, video_id]),
                     'title': episode.get('title'),
                     'description': episode.get('description'),
                     'season_id': season_id,
+                    'playlist_title': playlist_title,
                 })
                 entries.append(info)
 

From ef28e33249f650b3f8d40c3e62b9df2c6103b360 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 24 Feb 2021 22:29:35 +0100
Subject: [PATCH 244/860] [vvvvid] reduce season request payload size

---
 youtube_dl/extractor/vvvvid.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vvvvid.py b/youtube_dl/extractor/vvvvid.py
index d62404cf3..7c94c4ee2 100644
--- a/youtube_dl/extractor/vvvvid.py
+++ b/youtube_dl/extractor/vvvvid.py
@@ -75,12 +75,15 @@ class VVVVIDIE(InfoExtractor):
             'https://www.vvvvid.it/user/login',
             None, headers=self.geo_verification_headers())['data']['conn_id']
 
-    def _download_info(self, show_id, path, video_id, fatal=True):
+    def _download_info(self, show_id, path, video_id, fatal=True, query=None):
+        q = {
+            'conn_id': self._conn_id,
+        }
+        if query:
+            q.update(query)
         response = self._download_json(
             'https://www.vvvvid.it/vvvvid/ondemand/%s/%s' % (show_id, path),
-            video_id, headers=self.geo_verification_headers(), query={
-                'conn_id': self._conn_id,
-            }, fatal=fatal)
+            video_id, headers=self.geo_verification_headers(), query=q, fatal=fatal)
         if not (response or fatal):
             return
         if response.get('result') == 'error':
@@ -98,7 +101,8 @@ class VVVVIDIE(InfoExtractor):
         show_id, season_id, video_id = re.match(self._VALID_URL, url).groups()
 
         response = self._download_info(
-            show_id, 'season/%s' % season_id, video_id)
+            show_id, 'season/%s' % season_id,
+            video_id, query={'video_id': video_id})
 
         vid = int(video_id)
         video_data = list(filter(

From 3c58f9e0b9d8471212406e012727374db084932b Mon Sep 17 00:00:00 2001
From: Alexander Seiler <seileralex@gmail.com>
Date: Sat, 11 Nov 2017 19:30:10 +0100
Subject: [PATCH 245/860] [srgssr] improve extraction

- extract subtitle
- fix extraction for new videos
- update srf download domains

closes #14717
closes #14725
closes #27231
closes #28238
---
 youtube_dl/extractor/rts.py    |  15 ++-
 youtube_dl/extractor/srgssr.py | 206 +++++++++++++++++++++------------
 2 files changed, 143 insertions(+), 78 deletions(-)

diff --git a/youtube_dl/extractor/rts.py b/youtube_dl/extractor/rts.py
index 48f17b828..aed35f8a9 100644
--- a/youtube_dl/extractor/rts.py
+++ b/youtube_dl/extractor/rts.py
@@ -6,11 +6,12 @@ import re
 from .srgssr import SRGSSRIE
 from ..compat import compat_str
 from ..utils import (
+    determine_ext,
     int_or_none,
     parse_duration,
     parse_iso8601,
     unescapeHTML,
-    determine_ext,
+    urljoin,
 )
 
 
@@ -21,7 +22,7 @@ class RTSIE(SRGSSRIE):
     _TESTS = [
         {
             'url': 'http://www.rts.ch/archives/tv/divers/3449373-les-enfants-terribles.html',
-            'md5': 'ff7f8450a90cf58dacb64e29707b4a8e',
+            'md5': '753b877968ad8afaeddccc374d4256a5',
             'info_dict': {
                 'id': '3449373',
                 'display_id': 'les-enfants-terribles',
@@ -35,6 +36,7 @@ class RTSIE(SRGSSRIE):
                 'thumbnail': r're:^https?://.*\.image',
                 'view_count': int,
             },
+            'expected_warnings': ['Unable to download f4m manifest', 'Failed to download m3u8 information'],
         },
         {
             'url': 'http://www.rts.ch/emissions/passe-moi-les-jumelles/5624067-entre-ciel-et-mer.html',
@@ -63,11 +65,12 @@ class RTSIE(SRGSSRIE):
                 # m3u8 download
                 'skip_download': True,
             },
+            'expected_warnings': ['Unable to download f4m manifest', 'Failed to download m3u8 information'],
             'skip': 'Blocked outside Switzerland',
         },
         {
             'url': 'http://www.rts.ch/video/info/journal-continu/5745356-londres-cachee-par-un-epais-smog.html',
-            'md5': '1bae984fe7b1f78e94abc74e802ed99f',
+            'md5': '9bb06503773c07ce83d3cbd793cebb91',
             'info_dict': {
                 'id': '5745356',
                 'display_id': 'londres-cachee-par-un-epais-smog',
@@ -81,6 +84,7 @@ class RTSIE(SRGSSRIE):
                 'thumbnail': r're:^https?://.*\.image',
                 'view_count': int,
             },
+            'expected_warnings': ['Unable to download f4m manifest', 'Failed to download m3u8 information'],
         },
         {
             'url': 'http://www.rts.ch/audio/couleur3/programmes/la-belle-video-de-stephane-laurenceau/5706148-urban-hippie-de-damien-krisl-03-04-2014.html',
@@ -160,7 +164,7 @@ class RTSIE(SRGSSRIE):
         media_type = 'video' if 'video' in all_info else 'audio'
 
         # check for errors
-        self.get_media_data('rts', media_type, media_id)
+        self._get_media_data('rts', media_type, media_id)
 
         info = all_info['video']['JSONinfo'] if 'video' in all_info else all_info['audio']
 
@@ -194,6 +198,7 @@ class RTSIE(SRGSSRIE):
                     'tbr': extract_bitrate(format_url),
                 })
 
+        download_base = 'http://rtsww%s-d.rts.ch/' % ('-a' if media_type == 'audio' else '')
         for media in info.get('media', []):
             media_url = media.get('url')
             if not media_url or re.match(r'https?://', media_url):
@@ -205,7 +210,7 @@ class RTSIE(SRGSSRIE):
                 format_id += '-%dk' % rate
             formats.append({
                 'format_id': format_id,
-                'url': 'http://download-video.rts.ch/' + media_url,
+                'url': urljoin(download_base, media_url),
                 'tbr': rate or extract_bitrate(media_url),
             })
 
diff --git a/youtube_dl/extractor/srgssr.py b/youtube_dl/extractor/srgssr.py
index f63a1359a..ac018e740 100644
--- a/youtube_dl/extractor/srgssr.py
+++ b/youtube_dl/extractor/srgssr.py
@@ -4,16 +4,32 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_urllib_parse_urlparse
 from ..utils import (
     ExtractorError,
+    float_or_none,
+    int_or_none,
     parse_iso8601,
     qualities,
+    try_get,
 )
 
 
 class SRGSSRIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://tp\.srgssr\.ch/p(?:/[^/]+)+\?urn=urn|srgssr):(?P<bu>srf|rts|rsi|rtr|swi):(?:[^:]+:)?(?P<type>video|audio):(?P<id>[0-9a-f\-]{36}|\d+)'
+    _VALID_URL = r'''(?x)
+                    (?:
+                        https?://tp\.srgssr\.ch/p(?:/[^/]+)+\?urn=urn|
+                        srgssr
+                    ):
+                    (?P<bu>
+                        srf|rts|rsi|rtr|swi
+                    ):(?:[^:]+:)?
+                    (?P<type>
+                        video|audio
+                    ):
+                    (?P<id>
+                        [0-9a-f\-]{36}|\d+
+                    )
+                    '''
     _GEO_BYPASS = False
     _GEO_COUNTRIES = ['CH']
 
@@ -25,25 +41,39 @@ class SRGSSRIE(InfoExtractor):
         'LEGAL': 'The video cannot be transmitted for legal reasons.',
         'STARTDATE': 'This video is not yet available. Please try again later.',
     }
+    _DEFAULT_LANGUAGE_CODES = {
+        'srf': 'de',
+        'rts': 'fr',
+        'rsi': 'it',
+        'rtr': 'rm',
+        'swi': 'en',
+    }
 
     def _get_tokenized_src(self, url, video_id, format_id):
-        sp = compat_urllib_parse_urlparse(url).path.split('/')
         token = self._download_json(
-            'http://tp.srgssr.ch/akahd/token?acl=/%s/%s/*' % (sp[1], sp[2]),
+            'http://tp.srgssr.ch/akahd/token?acl=*',
             video_id, 'Downloading %s token' % format_id, fatal=False) or {}
-        auth_params = token.get('token', {}).get('authparams')
+        auth_params = try_get(token, lambda x: x['token']['authparams'])
         if auth_params:
-            url += '?' + auth_params
+            url += ('?' if '?' not in url else '&') + auth_params
         return url
 
-    def get_media_data(self, bu, media_type, media_id):
-        media_data = self._download_json(
-            'http://il.srgssr.ch/integrationlayer/1.0/ue/%s/%s/play/%s.json' % (bu, media_type, media_id),
-            media_id)[media_type.capitalize()]
+    def _get_media_data(self, bu, media_type, media_id):
+        query = {'onlyChapters': True} if media_type == 'video' else {}
+        full_media_data = self._download_json(
+            'https://il.srgssr.ch/integrationlayer/2.0/%s/mediaComposition/%s/%s.json'
+            % (bu, media_type, media_id),
+            media_id, query=query)['chapterList']
+        try:
+            media_data = next(
+                x for x in full_media_data if x.get('id') == media_id)
+        except StopIteration:
+            raise ExtractorError('No media information found')
 
-        if media_data.get('block') and media_data['block'] in self._ERRORS:
-            message = self._ERRORS[media_data['block']]
-            if media_data['block'] == 'GEOBLOCK':
+        block_reason = media_data.get('blockReason')
+        if block_reason and block_reason in self._ERRORS:
+            message = self._ERRORS[block_reason]
+            if block_reason == 'GEOBLOCK':
                 self.raise_geo_restricted(
                     msg=message, countries=self._GEO_COUNTRIES)
             raise ExtractorError(
@@ -53,53 +83,75 @@ class SRGSSRIE(InfoExtractor):
 
     def _real_extract(self, url):
         bu, media_type, media_id = re.match(self._VALID_URL, url).groups()
+        media_data = self._get_media_data(bu, media_type, media_id)
+        title = media_data['title']
 
-        media_data = self.get_media_data(bu, media_type, media_id)
-
-        metadata = media_data['AssetMetadatas']['AssetMetadata'][0]
-        title = metadata['title']
-        description = metadata.get('description')
-        created_date = media_data.get('createdDate') or metadata.get('createdDate')
-        timestamp = parse_iso8601(created_date)
-
-        thumbnails = [{
-            'id': image.get('id'),
-            'url': image['url'],
-        } for image in media_data.get('Image', {}).get('ImageRepresentations', {}).get('ImageRepresentation', [])]
-
-        preference = qualities(['LQ', 'MQ', 'SD', 'HQ', 'HD'])
         formats = []
-        for source in media_data.get('Playlists', {}).get('Playlist', []) + media_data.get('Downloads', {}).get('Download', []):
-            protocol = source.get('@protocol')
-            for asset in source['url']:
-                asset_url = asset['text']
-                quality = asset['@quality']
-                format_id = '%s-%s' % (protocol, quality)
-                if protocol.startswith('HTTP-HDS') or protocol.startswith('HTTP-HLS'):
-                    asset_url = self._get_tokenized_src(asset_url, media_id, format_id)
-                    if protocol.startswith('HTTP-HDS'):
-                        formats.extend(self._extract_f4m_formats(
-                            asset_url + ('?' if '?' not in asset_url else '&') + 'hdcore=3.4.0',
-                            media_id, f4m_id=format_id, fatal=False))
-                    elif protocol.startswith('HTTP-HLS'):
-                        formats.extend(self._extract_m3u8_formats(
-                            asset_url, media_id, 'mp4', 'm3u8_native',
-                            m3u8_id=format_id, fatal=False))
-                else:
-                    formats.append({
-                        'format_id': format_id,
-                        'url': asset_url,
-                        'preference': preference(quality),
-                        'ext': 'flv' if protocol == 'RTMP' else None,
-                    })
+        q = qualities(['SD', 'HD'])
+        for source in (media_data.get('resourceList') or []):
+            format_url = source.get('url')
+            if not format_url:
+                continue
+            protocol = source.get('protocol')
+            quality = source.get('quality')
+            format_id = []
+            for e in (protocol, source.get('encoding'), quality):
+                if e:
+                    format_id.append(e)
+            format_id = '-'.join(format_id)
+
+            if protocol in ('HDS', 'HLS'):
+                if source.get('tokenType') == 'AKAMAI':
+                    format_url = self._get_tokenized_src(
+                        format_url, media_id, format_id)
+                    formats.extend(self._extract_akamai_formats(
+                        format_url, media_id))
+                elif protocol == 'HLS':
+                    formats.extend(self._extract_m3u8_formats(
+                        format_url, media_id, 'mp4', 'm3u8_native',
+                        m3u8_id=format_id, fatal=False))
+            elif protocol in ('HTTP', 'HTTPS'):
+                formats.append({
+                    'format_id': format_id,
+                    'url': format_url,
+                    'quality': q(quality),
+                })
+
+        # This is needed because for audio medias the podcast url is usually
+        # always included, even if is only an audio segment and not the
+        # whole episode.
+        if int_or_none(media_data.get('position')) == 0:
+            for p in ('S', 'H'):
+                podcast_url = media_data.get('podcast%sdUrl' % p)
+                if not podcast_url:
+                    continue
+                quality = p + 'D'
+                formats.append({
+                    'format_id': 'PODCAST-' + quality,
+                    'url': podcast_url,
+                    'quality': q(quality),
+                })
         self._sort_formats(formats)
 
+        subtitles = {}
+        if media_type == 'video':
+            for sub in (media_data.get('subtitleList') or []):
+                sub_url = sub.get('url')
+                if not sub_url:
+                    continue
+                lang = sub.get('locale') or self._DEFAULT_LANGUAGE_CODES[bu]
+                subtitles.setdefault(lang, []).append({
+                    'url': sub_url,
+                })
+
         return {
             'id': media_id,
             'title': title,
-            'description': description,
-            'timestamp': timestamp,
-            'thumbnails': thumbnails,
+            'description': media_data.get('description'),
+            'timestamp': parse_iso8601(media_data.get('date')),
+            'thumbnail': media_data.get('imageUrl'),
+            'duration': float_or_none(media_data.get('duration'), 1000),
+            'subtitles': subtitles,
             'formats': formats,
         }
 
@@ -119,26 +171,17 @@ class SRGSSRPlayIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.srf.ch/play/tv/10vor10/video/snowden-beantragt-asyl-in-russland?id=28e1a57d-5b76-4399-8ab3-9097f071e6c5',
-        'md5': 'da6b5b3ac9fa4761a942331cef20fcb3',
+        'md5': '6db2226ba97f62ad42ce09783680046c',
         'info_dict': {
             'id': '28e1a57d-5b76-4399-8ab3-9097f071e6c5',
             'ext': 'mp4',
             'upload_date': '20130701',
             'title': 'Snowden beantragt Asyl in Russland',
-            'timestamp': 1372713995,
-        }
-    }, {
-        # No Speichern (Save) button
-        'url': 'http://www.srf.ch/play/tv/top-gear/video/jaguar-xk120-shadow-und-tornado-dampflokomotive?id=677f5829-e473-4823-ac83-a1087fe97faa',
-        'md5': '0a274ce38fda48c53c01890651985bc6',
-        'info_dict': {
-            'id': '677f5829-e473-4823-ac83-a1087fe97faa',
-            'ext': 'flv',
-            'upload_date': '20130710',
-            'title': 'Jaguar XK120, Shadow und Tornado-Dampflokomotive',
-            'description': 'md5:88604432b60d5a38787f152dec89cd56',
-            'timestamp': 1373493600,
+            'timestamp': 1372708215,
+            'duration': 113.827,
+            'thumbnail': r're:^https?://.*1383719781\.png$',
         },
+        'expected_warnings': ['Unable to download f4m manifest'],
     }, {
         'url': 'http://www.rtr.ch/play/radio/actualitad/audio/saira-tujetsch-tuttina-cuntinuar-cun-sedrun-muster-turissem?id=63cb0778-27f8-49af-9284-8c7a8c6d15fc',
         'info_dict': {
@@ -146,7 +189,8 @@ class SRGSSRPlayIE(InfoExtractor):
             'ext': 'mp3',
             'upload_date': '20151013',
             'title': 'Saira: Tujetsch - tuttina cuntinuar cun Sedrun Mustér Turissem',
-            'timestamp': 1444750398,
+            'timestamp': 1444709160,
+            'duration': 336.816,
         },
         'params': {
             # rtmp download
@@ -159,19 +203,32 @@ class SRGSSRPlayIE(InfoExtractor):
             'id': '6348260',
             'display_id': '6348260',
             'ext': 'mp4',
-            'duration': 1796,
+            'duration': 1796.76,
             'title': 'Le 19h30',
-            'description': '',
-            'uploader': '19h30',
             'upload_date': '20141201',
             'timestamp': 1417458600,
             'thumbnail': r're:^https?://.*\.image',
-            'view_count': int,
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         }
+    }, {
+        'url': 'http://play.swissinfo.ch/play/tv/business/video/why-people-were-against-tax-reforms?id=42960270',
+        'info_dict': {
+            'id': '42960270',
+            'ext': 'mp4',
+            'title': 'Why people were against tax reforms',
+            'description': 'md5:7ac442c558e9630e947427469c4b824d',
+            'duration': 94.0,
+            'upload_date': '20170215',
+            'timestamp': 1487173560,
+            'thumbnail': r're:https?://www\.swissinfo\.ch/srgscalableimage/42961964',
+            'subtitles': 'count:9',
+        },
+        'params': {
+            'skip_download': True,
+        }
     }, {
         'url': 'https://www.srf.ch/play/tv/popupvideoplayer?id=c4dba0ca-e75b-43b2-a34f-f708a4932e01',
         'only_matching': True,
@@ -181,6 +238,10 @@ class SRGSSRPlayIE(InfoExtractor):
     }, {
         'url': 'https://www.rts.ch/play/tv/19h30/video/le-19h30?urn=urn:rts:video:6348260',
         'only_matching': True,
+    }, {
+        # audio segment, has podcastSdUrl of the full episode
+        'url': 'https://www.srf.ch/play/radio/popupaudioplayer?id=50b20dc8-f05b-4972-bf03-e438ff2833eb',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -188,5 +249,4 @@ class SRGSSRPlayIE(InfoExtractor):
         bu = mobj.group('bu')
         media_type = mobj.group('type') or mobj.group('type_2')
         media_id = mobj.group('id')
-        # other info can be extracted from url + '&layout=json'
         return self.url_result('srgssr:%s:%s:%s' % (bu[:3], media_type, media_id), 'SRGSSR')

From 678d46f6bbcc8426723d48c49eb25cf202753245 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 28 Feb 2021 10:42:41 +0100
Subject: [PATCH 246/860] [bandaichannel] Add new extractor(closes #21404)

---
 youtube_dl/extractor/bandaichannel.py | 37 +++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py    |  1 +
 2 files changed, 38 insertions(+)
 create mode 100644 youtube_dl/extractor/bandaichannel.py

diff --git a/youtube_dl/extractor/bandaichannel.py b/youtube_dl/extractor/bandaichannel.py
new file mode 100644
index 000000000..d67285913
--- /dev/null
+++ b/youtube_dl/extractor/bandaichannel.py
@@ -0,0 +1,37 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .brightcove import BrightcoveNewIE
+from ..utils import extract_attributes
+
+
+class BandaiChannelIE(BrightcoveNewIE):
+    IE_NAME = 'bandaichannel'
+    _VALID_URL = r'https?://(?:www\.)?b-ch\.com/titles/(?P<id>\d+/\d+)'
+    _TESTS = [{
+        'url': 'https://www.b-ch.com/titles/514/001',
+        'md5': 'a0f2d787baa5729bed71108257f613a4',
+        'info_dict': {
+            'id': '6128044564001',
+            'ext': 'mp4',
+            'title': 'メタルファイターMIKU 第1話',
+            'timestamp': 1580354056,
+            'uploader_id': '5797077852001',
+            'upload_date': '20200130',
+            'duration': 1387.733,
+        },
+        'params': {
+            'format': 'bestvideo',
+            'skip_download': True,
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        attrs = extract_attributes(self._search_regex(
+            r'(<video-js[^>]+\bid="bcplayer"[^>]*>)', webpage, 'player'))
+        bc = self._download_json(
+            'https://pbifcd.b-ch.com/v1/playbackinfo/ST/70/' + attrs['data-info'],
+            video_id, headers={'X-API-KEY': attrs['data-auth'].strip()})['bc']
+        return self._parse_brightcove_metadata(bc, bc['id'])
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index dc6a06771..07a8af055 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -90,6 +90,7 @@ from .awaan import (
 )
 from .azmedien import AZMedienIE
 from .baidu import BaiduVideoIE
+from .bandaichannel import BandaiChannelIE
 from .bandcamp import BandcampIE, BandcampAlbumIE, BandcampWeeklyIE
 from .bbc import (
     BBCCoUkIE,

From 38fe5e239ad602b32c111f40ad7c51b3e029be3c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 28 Feb 2021 12:31:18 +0100
Subject: [PATCH 247/860] [urplay] fix episode data extraction(closes #28292)

---
 youtube_dl/extractor/urplay.py | 15 +++++++++++++--
 1 file changed, 13 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/urplay.py b/youtube_dl/extractor/urplay.py
index 5452c7ca1..d6c79147e 100644
--- a/youtube_dl/extractor/urplay.py
+++ b/youtube_dl/extractor/urplay.py
@@ -21,6 +21,11 @@ class URPlayIE(InfoExtractor):
             'description': 'md5:5344508a52aa78c1ced6c1b8b9e44e9a',
             'timestamp': 1513292400,
             'upload_date': '20171214',
+            'series': 'UR Samtiden - Livet, universum och rymdens märkliga musik',
+            'duration': 2269,
+            'categories': ['Kultur & historia'],
+            'tags': ['Kritiskt tänkande', 'Vetenskap', 'Vetenskaplig verksamhet'],
+            'episode': 'Om vetenskap, kritiskt tänkande och motstånd',
         },
     }, {
         'url': 'https://urskola.se/Produkter/190031-Tripp-Trapp-Trad-Sovkudde',
@@ -31,6 +36,10 @@ class URPlayIE(InfoExtractor):
             'description': 'md5:b86bffdae04a7e9379d1d7e5947df1d1',
             'timestamp': 1440086400,
             'upload_date': '20150820',
+            'series': 'Tripp, Trapp, Träd',
+            'duration': 865,
+            'tags': ['Sova'],
+            'episode': 'Sovkudde',
         },
     }, {
         'url': 'http://urskola.se/Produkter/155794-Smasagor-meankieli-Grodan-i-vida-varlden',
@@ -41,9 +50,11 @@ class URPlayIE(InfoExtractor):
         video_id = self._match_id(url)
         url = url.replace('skola.se/Produkter', 'play.se/program')
         webpage = self._download_webpage(url, video_id)
-        urplayer_data = self._parse_json(self._html_search_regex(
+        vid = int(video_id)
+        accessible_episodes = self._parse_json(self._html_search_regex(
             r'data-react-class="routes/Product/components/ProgramContainer/ProgramContainer"[^>]+data-react-props="({.+?})"',
-            webpage, 'urplayer data'), video_id)['accessibleEpisodes'][0]
+            webpage, 'urplayer data'), video_id)['accessibleEpisodes']
+        urplayer_data = next(e for e in accessible_episodes if e.get('id') == vid)
         episode = urplayer_data['title']
         raw_streaming_info = urplayer_data['streamingInfo']['raw']
         host = self._download_json(

From bee618268014480bb3dd7887986b456c8e9c0236 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 1 Mar 2021 14:00:03 +0100
Subject: [PATCH 248/860] [stretchinternet] Fix extraction(closes #28297)

---
 youtube_dl/extractor/stretchinternet.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/stretchinternet.py b/youtube_dl/extractor/stretchinternet.py
index 4dbead2ba..ec08eae55 100644
--- a/youtube_dl/extractor/stretchinternet.py
+++ b/youtube_dl/extractor/stretchinternet.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import int_or_none
 
 
 class StretchInternetIE(InfoExtractor):
@@ -11,22 +10,28 @@ class StretchInternetIE(InfoExtractor):
         'info_dict': {
             'id': '573272',
             'ext': 'mp4',
-            'title': 'University of Mary Wrestling vs. Upper Iowa',
-            'timestamp': 1575668361,
-            'upload_date': '20191206',
+            'title': 'UNIVERSITY OF MARY WRESTLING VS UPPER IOWA',
+            # 'timestamp': 1575668361,
+            # 'upload_date': '20191206',
+            'uploader_id': '99997',
         }
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
+        media_url = self._download_json(
+            'https://core.stretchlive.com/trinity/event/tcg/' + video_id,
+            video_id)[0]['media'][0]['url']
         event = self._download_json(
-            'https://api.stretchinternet.com/trinity/event/tcg/' + video_id,
-            video_id)[0]
+            'https://neo-client.stretchinternet.com/portal-ws/getEvent.json',
+            video_id, query={'eventID': video_id, 'token': 'asdf'})['event']
 
         return {
             'id': video_id,
             'title': event['title'],
-            'timestamp': int_or_none(event.get('dateCreated'), 1000),
-            'url': 'https://' + event['media'][0]['url'],
+            # TODO: parse US timezone abbreviations
+            # 'timestamp': event.get('dateTimeString'),
+            'url': 'https://' + media_url,
+            'uploader_id': event.get('ownerID'),
         }

From 3fb14cd214fdadfae195745b26498e012f78be8e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Mar 2021 06:03:17 +0700
Subject: [PATCH 249/860] [zdf] Rework extractors (closes #11606, closes
 #13473, closes #17354, closes #21185, closes #26711, closes #27068, closes
 #27930, closes #28198, closes #28199, closes #28274)

* Generalize unique video ids for zdf based extractors
* Improve extraction
* Fix 3sat and phoenix
---
 youtube_dl/extractor/dreisat.py | 220 +++++---------------------------
 youtube_dl/extractor/phoenix.py | 149 ++++++++++++++++-----
 youtube_dl/extractor/zdf.py     | 192 ++++++++++++++++++----------
 3 files changed, 276 insertions(+), 285 deletions(-)

diff --git a/youtube_dl/extractor/dreisat.py b/youtube_dl/extractor/dreisat.py
index 848d387d1..5a07c18f4 100644
--- a/youtube_dl/extractor/dreisat.py
+++ b/youtube_dl/extractor/dreisat.py
@@ -1,193 +1,43 @@
 from __future__ import unicode_literals
 
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    int_or_none,
-    unified_strdate,
-    xpath_text,
-    determine_ext,
-    float_or_none,
-    ExtractorError,
-)
+from .zdf import ZDFIE
 
 
-class DreiSatIE(InfoExtractor):
+class DreiSatIE(ZDFIE):
     IE_NAME = '3sat'
-    _GEO_COUNTRIES = ['DE']
-    _VALID_URL = r'https?://(?:www\.)?3sat\.de/mediathek/(?:(?:index|mediathek)\.php)?\?(?:(?:mode|display)=[^&]+&)*obj=(?P<id>[0-9]+)'
-    _TESTS = [
-        {
-            'url': 'http://www.3sat.de/mediathek/index.php?mode=play&obj=45918',
-            'md5': 'be37228896d30a88f315b638900a026e',
-            'info_dict': {
-                'id': '45918',
-                'ext': 'mp4',
-                'title': 'Waidmannsheil',
-                'description': 'md5:cce00ca1d70e21425e72c86a98a56817',
-                'uploader': 'SCHWEIZWEIT',
-                'uploader_id': '100000210',
-                'upload_date': '20140913'
-            },
-            'params': {
-                'skip_download': True,  # m3u8 downloads
-            }
+    _VALID_URL = r'https?://(?:www\.)?3sat\.de/(?:[^/]+/)*(?P<id>[^/?#&]+)\.html'
+    _TESTS = [{
+        # Same as https://www.zdf.de/dokumentation/ab-18/10-wochen-sommer-102.html
+        'url': 'https://www.3sat.de/film/ab-18/10-wochen-sommer-108.html',
+        'md5': '0aff3e7bc72c8813f5e0fae333316a1d',
+        'info_dict': {
+            'id': '141007_ab18_10wochensommer_film',
+            'ext': 'mp4',
+            'title': 'Ab 18! - 10 Wochen Sommer',
+            'description': 'md5:8253f41dc99ce2c3ff892dac2d65fe26',
+            'duration': 2660,
+            'timestamp': 1608604200,
+            'upload_date': '20201222',
         },
-        {
-            'url': 'http://www.3sat.de/mediathek/mediathek.php?mode=play&obj=51066',
-            'only_matching': True,
+    }, {
+        'url': 'https://www.3sat.de/gesellschaft/schweizweit/waidmannsheil-100.html',
+        'info_dict': {
+            'id': '140913_sendung_schweizweit',
+            'ext': 'mp4',
+            'title': 'Waidmannsheil',
+            'description': 'md5:cce00ca1d70e21425e72c86a98a56817',
+            'timestamp': 1410623100,
+            'upload_date': '20140913'
         },
-    ]
-
-    def _parse_smil_formats(self, smil, smil_url, video_id, namespace=None, f4m_params=None, transform_rtmp_url=None):
-        param_groups = {}
-        for param_group in smil.findall(self._xpath_ns('./head/paramGroup', namespace)):
-            group_id = param_group.get(self._xpath_ns(
-                'id', 'http://www.w3.org/XML/1998/namespace'))
-            params = {}
-            for param in param_group:
-                params[param.get('name')] = param.get('value')
-            param_groups[group_id] = params
-
-        formats = []
-        for video in smil.findall(self._xpath_ns('.//video', namespace)):
-            src = video.get('src')
-            if not src:
-                continue
-            bitrate = int_or_none(self._search_regex(r'_(\d+)k', src, 'bitrate', None)) or float_or_none(video.get('system-bitrate') or video.get('systemBitrate'), 1000)
-            group_id = video.get('paramGroup')
-            param_group = param_groups[group_id]
-            for proto in param_group['protocols'].split(','):
-                formats.append({
-                    'url': '%s://%s' % (proto, param_group['host']),
-                    'app': param_group['app'],
-                    'play_path': src,
-                    'ext': 'flv',
-                    'format_id': '%s-%d' % (proto, bitrate),
-                    'tbr': bitrate,
-                })
-        self._sort_formats(formats)
-        return formats
-
-    def extract_from_xml_url(self, video_id, xml_url):
-        doc = self._download_xml(
-            xml_url, video_id,
-            note='Downloading video info',
-            errnote='Failed to download video info')
-
-        status_code = xpath_text(doc, './status/statuscode')
-        if status_code and status_code != 'ok':
-            if status_code == 'notVisibleAnymore':
-                message = 'Video %s is not available' % video_id
-            else:
-                message = '%s returned error: %s' % (self.IE_NAME, status_code)
-            raise ExtractorError(message, expected=True)
-
-        title = xpath_text(doc, './/information/title', 'title', True)
-
-        urls = []
-        formats = []
-        for fnode in doc.findall('.//formitaeten/formitaet'):
-            video_url = xpath_text(fnode, 'url')
-            if not video_url or video_url in urls:
-                continue
-            urls.append(video_url)
-
-            is_available = 'http://www.metafilegenerator' not in video_url
-            geoloced = 'static_geoloced_online' in video_url
-            if not is_available or geoloced:
-                continue
-
-            format_id = fnode.attrib['basetype']
-            format_m = re.match(r'''(?x)
-                (?P<vcodec>[^_]+)_(?P<acodec>[^_]+)_(?P<container>[^_]+)_
-                (?P<proto>[^_]+)_(?P<index>[^_]+)_(?P<indexproto>[^_]+)
-            ''', format_id)
-
-            ext = determine_ext(video_url, None) or format_m.group('container')
-
-            if ext == 'meta':
-                continue
-            elif ext == 'smil':
-                formats.extend(self._extract_smil_formats(
-                    video_url, video_id, fatal=False))
-            elif ext == 'm3u8':
-                # the certificates are misconfigured (see
-                # https://github.com/ytdl-org/youtube-dl/issues/8665)
-                if video_url.startswith('https://'):
-                    continue
-                formats.extend(self._extract_m3u8_formats(
-                    video_url, video_id, 'mp4', 'm3u8_native',
-                    m3u8_id=format_id, fatal=False))
-            elif ext == 'f4m':
-                formats.extend(self._extract_f4m_formats(
-                    video_url, video_id, f4m_id=format_id, fatal=False))
-            else:
-                quality = xpath_text(fnode, './quality')
-                if quality:
-                    format_id += '-' + quality
-
-                abr = int_or_none(xpath_text(fnode, './audioBitrate'), 1000)
-                vbr = int_or_none(xpath_text(fnode, './videoBitrate'), 1000)
-
-                tbr = int_or_none(self._search_regex(
-                    r'_(\d+)k', video_url, 'bitrate', None))
-                if tbr and vbr and not abr:
-                    abr = tbr - vbr
-
-                formats.append({
-                    'format_id': format_id,
-                    'url': video_url,
-                    'ext': ext,
-                    'acodec': format_m.group('acodec'),
-                    'vcodec': format_m.group('vcodec'),
-                    'abr': abr,
-                    'vbr': vbr,
-                    'tbr': tbr,
-                    'width': int_or_none(xpath_text(fnode, './width')),
-                    'height': int_or_none(xpath_text(fnode, './height')),
-                    'filesize': int_or_none(xpath_text(fnode, './filesize')),
-                    'protocol': format_m.group('proto').lower(),
-                })
-
-        geolocation = xpath_text(doc, './/details/geolocation')
-        if not formats and geolocation and geolocation != 'none':
-            self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
-
-        self._sort_formats(formats)
-
-        thumbnails = []
-        for node in doc.findall('.//teaserimages/teaserimage'):
-            thumbnail_url = node.text
-            if not thumbnail_url:
-                continue
-            thumbnail = {
-                'url': thumbnail_url,
-            }
-            thumbnail_key = node.get('key')
-            if thumbnail_key:
-                m = re.match('^([0-9]+)x([0-9]+)$', thumbnail_key)
-                if m:
-                    thumbnail['width'] = int(m.group(1))
-                    thumbnail['height'] = int(m.group(2))
-            thumbnails.append(thumbnail)
-
-        upload_date = unified_strdate(xpath_text(doc, './/details/airtime'))
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': xpath_text(doc, './/information/detail'),
-            'duration': int_or_none(xpath_text(doc, './/details/lengthSec')),
-            'thumbnails': thumbnails,
-            'uploader': xpath_text(doc, './/details/originChannelTitle'),
-            'uploader_id': xpath_text(doc, './/details/originChannelId'),
-            'upload_date': upload_date,
-            'formats': formats,
+        'params': {
+            'skip_download': True,
         }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        details_url = 'http://www.3sat.de/mediathek/xmlservice/web/beitragsDetails?id=%s' % video_id
-        return self.extract_from_xml_url(video_id, details_url)
+    }, {
+        # Same as https://www.zdf.de/filme/filme-sonstige/der-hauptmann-112.html
+        'url': 'https://www.3sat.de/film/spielfilm/der-hauptmann-100.html',
+        'only_matching': True,
+    }, {
+        # Same as https://www.zdf.de/wissen/nano/nano-21-mai-2019-102.html, equal media ids
+        'url': 'https://www.3sat.de/wissen/nano/nano-21-mai-2019-102.html',
+        'only_matching': True,
+    }]
diff --git a/youtube_dl/extractor/phoenix.py b/youtube_dl/extractor/phoenix.py
index e435c28e1..dbbfce983 100644
--- a/youtube_dl/extractor/phoenix.py
+++ b/youtube_dl/extractor/phoenix.py
@@ -1,45 +1,128 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
-from .dreisat import DreiSatIE
+import re
+
+from .youtube import YoutubeIE
+from .zdf import ZDFBaseIE
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    merge_dicts,
+    unified_timestamp,
+    xpath_text,
+)
 
 
-class PhoenixIE(DreiSatIE):
+class PhoenixIE(ZDFBaseIE):
     IE_NAME = 'phoenix.de'
-    _VALID_URL = r'''(?x)https?://(?:www\.)?phoenix\.de/content/
-        (?:
-            phoenix/die_sendungen/(?:[^/]+/)?
-        )?
-        (?P<id>[0-9]+)'''
-    _TESTS = [
-        {
-            'url': 'http://www.phoenix.de/content/884301',
-            'md5': 'ed249f045256150c92e72dbb70eadec6',
-            'info_dict': {
-                'id': '884301',
-                'ext': 'mp4',
-                'title': 'Michael Krons mit Hans-Werner Sinn',
-                'description': 'Im Dialog - Sa. 25.10.14, 00.00 - 00.35 Uhr',
-                'upload_date': '20141025',
-                'uploader': 'Im Dialog',
-            }
+    _VALID_URL = r'https?://(?:www\.)?phoenix\.de/(?:[^/]+/)*[^/?#&]*-a-(?P<id>\d+)\.html'
+    _TESTS = [{
+        # Same as https://www.zdf.de/politik/phoenix-sendungen/wohin-fuehrt-der-protest-in-der-pandemie-100.html
+        'url': 'https://www.phoenix.de/sendungen/ereignisse/corona-nachgehakt/wohin-fuehrt-der-protest-in-der-pandemie-a-2050630.html',
+        'md5': '34ec321e7eb34231fd88616c65c92db0',
+        'info_dict': {
+            'id': '210222_phx_nachgehakt_corona_protest',
+            'ext': 'mp4',
+            'title': 'Wohin führt der Protest in der Pandemie?',
+            'description': 'md5:7d643fe7f565e53a24aac036b2122fbd',
+            'duration': 1691,
+            'timestamp': 1613906100,
+            'upload_date': '20210221',
+            'uploader': 'Phoenix',
+            'channel': 'corona nachgehakt',
         },
-        {
-            'url': 'http://www.phoenix.de/content/phoenix/die_sendungen/869815',
-            'only_matching': True,
+    }, {
+        # Youtube embed
+        'url': 'https://www.phoenix.de/sendungen/gespraeche/phoenix-streitgut-brennglas-corona-a-1965505.html',
+        'info_dict': {
+            'id': 'hMQtqFYjomk',
+            'ext': 'mp4',
+            'title': 'phoenix streitgut: Brennglas Corona - Wie gerecht ist unsere Gesellschaft?',
+            'description': 'md5:ac7a02e2eb3cb17600bc372e4ab28fdd',
+            'duration': 3509,
+            'upload_date': '20201219',
+            'uploader': 'phoenix',
+            'uploader_id': 'phoenix',
         },
-        {
-            'url': 'http://www.phoenix.de/content/phoenix/die_sendungen/diskussionen/928234',
-            'only_matching': True,
+        'params': {
+            'skip_download': True,
         },
-    ]
+    }, {
+        'url': 'https://www.phoenix.de/entwicklungen-in-russland-a-2044720.html',
+        'only_matching': True,
+    }, {
+        # no media
+        'url': 'https://www.phoenix.de/sendungen/dokumentationen/mit-dem-jumbo-durch-die-nacht-a-89625.html',
+        'only_matching': True,
+    }, {
+        # Same as https://www.zdf.de/politik/phoenix-sendungen/die-gesten-der-maechtigen-100.html
+        'url': 'https://www.phoenix.de/sendungen/dokumentationen/gesten-der-maechtigen-i-a-89468.html?ref=suche',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
+        article_id = self._match_id(url)
 
-        internal_id = self._search_regex(
-            r'<div class="phx_vod" id="phx_vod_([0-9]+)"',
-            webpage, 'internal video ID')
+        article = self._download_json(
+            'https://www.phoenix.de/response/id/%s' % article_id, article_id,
+            'Downloading article JSON')
 
-        api_url = 'http://www.phoenix.de/php/mediaplayer/data/beitrags_details.php?ak=web&id=%s' % internal_id
-        return self.extract_from_xml_url(video_id, api_url)
+        video = article['absaetze'][0]
+        title = video.get('titel') or article.get('subtitel')
+
+        if video.get('typ') == 'video-youtube':
+            video_id = video['id']
+            return self.url_result(
+                video_id, ie=YoutubeIE.ie_key(), video_id=video_id,
+                video_title=title)
+
+        video_id = compat_str(video.get('basename') or video.get('content'))
+
+        details = self._download_xml(
+            'https://www.phoenix.de/php/mediaplayer/data/beitrags_details.php',
+            video_id, 'Downloading details XML', query={
+                'ak': 'web',
+                'ptmd': 'true',
+                'id': video_id,
+                'profile': 'player2',
+            })
+
+        title = title or xpath_text(
+            details, './/information/title', 'title', fatal=True)
+        content_id = xpath_text(
+            details, './/video/details/basename', 'content id', fatal=True)
+
+        info = self._extract_ptmd(
+            'https://tmd.phoenix.de/tmd/2/ngplayer_2_3/vod/ptmd/phoenix/%s' % content_id,
+            content_id, None, url)
+
+        timestamp = unified_timestamp(xpath_text(details, './/details/airtime'))
+
+        thumbnails = []
+        for node in details.findall('.//teaserimages/teaserimage'):
+            thumbnail_url = node.text
+            if not thumbnail_url:
+                continue
+            thumbnail = {
+                'url': thumbnail_url,
+            }
+            thumbnail_key = node.get('key')
+            if thumbnail_key:
+                m = re.match('^([0-9]+)x([0-9]+)$', thumbnail_key)
+                if m:
+                    thumbnail['width'] = int(m.group(1))
+                    thumbnail['height'] = int(m.group(2))
+            thumbnails.append(thumbnail)
+
+        return merge_dicts(info, {
+            'id': content_id,
+            'title': title,
+            'description': xpath_text(details, './/information/detail'),
+            'duration': int_or_none(xpath_text(details, './/details/lengthSec')),
+            'thumbnails': thumbnails,
+            'timestamp': timestamp,
+            'uploader': xpath_text(details, './/details/channel'),
+            'uploader_id': xpath_text(details, './/details/originChannelId'),
+            'channel': xpath_text(details, './/details/originChannelTitle'),
+        })
diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 5ed2946c2..4dd56f66d 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -7,7 +7,9 @@ from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
     determine_ext,
+    float_or_none,
     int_or_none,
+    merge_dicts,
     NO_DEFAULT,
     orderedSet,
     parse_codecs,
@@ -21,49 +23,17 @@ from ..utils import (
 
 
 class ZDFBaseIE(InfoExtractor):
-    def _call_api(self, url, player, referrer, video_id, item):
-        return self._download_json(
-            url, video_id, 'Downloading JSON %s' % item,
-            headers={
-                'Referer': referrer,
-                'Api-Auth': 'Bearer %s' % player['apiToken'],
-            })
-
-    def _extract_player(self, webpage, video_id, fatal=True):
-        return self._parse_json(
-            self._search_regex(
-                r'(?s)data-zdfplayer-jsb=(["\'])(?P<json>{.+?})\1', webpage,
-                'player JSON', default='{}' if not fatal else NO_DEFAULT,
-                group='json'),
-            video_id)
-
-
-class ZDFIE(ZDFBaseIE):
-    _VALID_URL = r'https?://www\.zdf\.de/(?:[^/]+/)*(?P<id>[^/?]+)\.html'
-    _QUALITIES = ('auto', 'low', 'med', 'high', 'veryhigh', 'hd')
     _GEO_COUNTRIES = ['DE']
+    _QUALITIES = ('auto', 'low', 'med', 'high', 'veryhigh', 'hd')
 
-    _TESTS = [{
-        'url': 'https://www.zdf.de/dokumentation/terra-x/die-magie-der-farben-von-koenigspurpur-und-jeansblau-100.html',
-        'info_dict': {
-            'id': 'die-magie-der-farben-von-koenigspurpur-und-jeansblau-100',
-            'ext': 'mp4',
-            'title': 'Die Magie der Farben (2/2)',
-            'description': 'md5:a89da10c928c6235401066b60a6d5c1a',
-            'duration': 2615,
-            'timestamp': 1465021200,
-            'upload_date': '20160604',
-        },
-    }, {
-        'url': 'https://www.zdf.de/service-und-hilfe/die-neue-zdf-mediathek/zdfmediathek-trailer-100.html',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.zdf.de/filme/taunuskrimi/die-lebenden-und-die-toten-1---ein-taunuskrimi-100.html',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.zdf.de/dokumentation/planet-e/planet-e-uebersichtsseite-weitere-dokumentationen-von-planet-e-100.html',
-        'only_matching': True,
-    }]
+    def _call_api(self, url, video_id, item, api_token=None, referrer=None):
+        headers = {}
+        if api_token:
+            headers['Api-Auth'] = 'Bearer %s' % api_token
+        if referrer:
+            headers['Referer'] = referrer
+        return self._download_json(
+            url, video_id, 'Downloading JSON %s' % item, headers=headers)
 
     @staticmethod
     def _extract_subtitles(src):
@@ -109,20 +79,11 @@ class ZDFIE(ZDFBaseIE):
             })
             formats.append(f)
 
-    def _extract_entry(self, url, player, content, video_id):
-        title = content.get('title') or content['teaserHeadline']
-
-        t = content['mainVideoContent']['http://zdf.de/rels/target']
-
-        ptmd_path = t.get('http://zdf.de/rels/streams/ptmd')
-
-        if not ptmd_path:
-            ptmd_path = t[
-                'http://zdf.de/rels/streams/ptmd-template'].replace(
-                '{playerId}', 'ngplayer_2_4')
-
+    def _extract_ptmd(self, ptmd_url, video_id, api_token, referrer):
         ptmd = self._call_api(
-            urljoin(url, ptmd_path), player, url, video_id, 'metadata')
+            ptmd_url, video_id, 'metadata', api_token, referrer)
+
+        content_id = ptmd.get('basename') or ptmd_url.split('/')[-1]
 
         formats = []
         track_uris = set()
@@ -140,7 +101,7 @@ class ZDFIE(ZDFBaseIE):
                         continue
                     for track in tracks:
                         self._extract_format(
-                            video_id, formats, track_uris, {
+                            content_id, formats, track_uris, {
                                 'url': track.get('uri'),
                                 'type': f.get('type'),
                                 'mimeType': f.get('mimeType'),
@@ -149,6 +110,103 @@ class ZDFIE(ZDFBaseIE):
                             })
         self._sort_formats(formats)
 
+        duration = float_or_none(try_get(
+            ptmd, lambda x: x['attributes']['duration']['value']), scale=1000)
+
+        return {
+            'extractor_key': ZDFIE.ie_key(),
+            'id': content_id,
+            'duration': duration,
+            'formats': formats,
+            'subtitles': self._extract_subtitles(ptmd),
+        }
+
+    def _extract_player(self, webpage, video_id, fatal=True):
+        return self._parse_json(
+            self._search_regex(
+                r'(?s)data-zdfplayer-jsb=(["\'])(?P<json>{.+?})\1', webpage,
+                'player JSON', default='{}' if not fatal else NO_DEFAULT,
+                group='json'),
+            video_id)
+
+
+class ZDFIE(ZDFBaseIE):
+    _VALID_URL = r'https?://www\.zdf\.de/(?:[^/]+/)*(?P<id>[^/?#&]+)\.html'
+    _TESTS = [{
+        # Same as https://www.phoenix.de/sendungen/ereignisse/corona-nachgehakt/wohin-fuehrt-der-protest-in-der-pandemie-a-2050630.html
+        'url': 'https://www.zdf.de/politik/phoenix-sendungen/wohin-fuehrt-der-protest-in-der-pandemie-100.html',
+        'md5': '34ec321e7eb34231fd88616c65c92db0',
+        'info_dict': {
+            'id': '210222_phx_nachgehakt_corona_protest',
+            'ext': 'mp4',
+            'title': 'Wohin führt der Protest in der Pandemie?',
+            'description': 'md5:7d643fe7f565e53a24aac036b2122fbd',
+            'duration': 1691,
+            'timestamp': 1613948400,
+            'upload_date': '20210221',
+        },
+    }, {
+        # Same as https://www.3sat.de/film/ab-18/10-wochen-sommer-108.html
+        'url': 'https://www.zdf.de/dokumentation/ab-18/10-wochen-sommer-102.html',
+        'md5': '0aff3e7bc72c8813f5e0fae333316a1d',
+        'info_dict': {
+            'id': '141007_ab18_10wochensommer_film',
+            'ext': 'mp4',
+            'title': 'Ab 18! - 10 Wochen Sommer',
+            'description': 'md5:8253f41dc99ce2c3ff892dac2d65fe26',
+            'duration': 2660,
+            'timestamp': 1608604200,
+            'upload_date': '20201222',
+        },
+    }, {
+        'url': 'https://www.zdf.de/dokumentation/terra-x/die-magie-der-farben-von-koenigspurpur-und-jeansblau-100.html',
+        'info_dict': {
+            'id': '151025_magie_farben2_tex',
+            'ext': 'mp4',
+            'title': 'Die Magie der Farben (2/2)',
+            'description': 'md5:a89da10c928c6235401066b60a6d5c1a',
+            'duration': 2615,
+            'timestamp': 1465021200,
+            'upload_date': '20160604',
+        },
+    }, {
+        # Same as https://www.phoenix.de/sendungen/dokumentationen/gesten-der-maechtigen-i-a-89468.html?ref=suche
+        'url': 'https://www.zdf.de/politik/phoenix-sendungen/die-gesten-der-maechtigen-100.html',
+        'only_matching': True,
+    }, {
+        # Same as https://www.3sat.de/film/spielfilm/der-hauptmann-100.html
+        'url': 'https://www.zdf.de/filme/filme-sonstige/der-hauptmann-112.html',
+        'only_matching': True,
+    }, {
+        # Same as https://www.3sat.de/wissen/nano/nano-21-mai-2019-102.html, equal media ids
+        'url': 'https://www.zdf.de/wissen/nano/nano-21-mai-2019-102.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.zdf.de/service-und-hilfe/die-neue-zdf-mediathek/zdfmediathek-trailer-100.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.zdf.de/filme/taunuskrimi/die-lebenden-und-die-toten-1---ein-taunuskrimi-100.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.zdf.de/dokumentation/planet-e/planet-e-uebersichtsseite-weitere-dokumentationen-von-planet-e-100.html',
+        'only_matching': True,
+    }]
+
+    def _extract_entry(self, url, player, content, video_id):
+        title = content.get('title') or content['teaserHeadline']
+
+        t = content['mainVideoContent']['http://zdf.de/rels/target']
+
+        ptmd_path = t.get('http://zdf.de/rels/streams/ptmd')
+
+        if not ptmd_path:
+            ptmd_path = t[
+                'http://zdf.de/rels/streams/ptmd-template'].replace(
+                '{playerId}', 'ngplayer_2_4')
+
+        info = self._extract_ptmd(
+            urljoin(url, ptmd_path), video_id, player['apiToken'], url)
+
         thumbnails = []
         layouts = try_get(
             content, lambda x: x['teaserImageRef']['layouts'], dict)
@@ -169,33 +227,33 @@ class ZDFIE(ZDFBaseIE):
                     })
                 thumbnails.append(thumbnail)
 
-        return {
-            'id': video_id,
+        return merge_dicts(info, {
             'title': title,
             'description': content.get('leadParagraph') or content.get('teasertext'),
             'duration': int_or_none(t.get('duration')),
             'timestamp': unified_timestamp(content.get('editorialDate')),
             'thumbnails': thumbnails,
-            'subtitles': self._extract_subtitles(ptmd),
-            'formats': formats,
-        }
+        })
 
     def _extract_regular(self, url, player, video_id):
         content = self._call_api(
-            player['content'], player, url, video_id, 'content')
+            player['content'], video_id, 'content', player['apiToken'], url)
         return self._extract_entry(player['content'], player, content, video_id)
 
     def _extract_mobile(self, video_id):
-        document = self._download_json(
+        video = self._download_json(
             'https://zdf-cdn.live.cellular.de/mediathekV2/document/%s' % video_id,
-            video_id)['document']
+            video_id)
+
+        document = video['document']
 
         title = document['titel']
+        content_id = document['basename']
 
         formats = []
         format_urls = set()
         for f in document['formitaeten']:
-            self._extract_format(video_id, formats, format_urls, f)
+            self._extract_format(content_id, formats, format_urls, f)
         self._sort_formats(formats)
 
         thumbnails = []
@@ -213,12 +271,12 @@ class ZDFIE(ZDFBaseIE):
                     })
 
         return {
-            'id': video_id,
+            'id': content_id,
             'title': title,
             'description': document.get('beschreibung'),
             'duration': int_or_none(document.get('length')),
-            'timestamp': unified_timestamp(try_get(
-                document, lambda x: x['meta']['editorialDate'], compat_str)),
+            'timestamp': unified_timestamp(document.get('date')) or unified_timestamp(
+                try_get(video, lambda x: x['meta']['editorialDate'], compat_str)),
             'thumbnails': thumbnails,
             'subtitles': self._extract_subtitles(document),
             'formats': formats,

From 0002888627b40264994e8a37fc3a17cbd3551af6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Mar 2021 06:16:41 +0700
Subject: [PATCH 250/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 2912d776c..07f26f2cf 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,28 @@
+version <unreleased>
+
+Extractors
+* [zdf] Rework extractors (#11606, #13473, #17354, #21185, #26711, #27068,
+  #27930, #28198, #28199, #28274)
+    * Generalize cross-extractor video ids for zdf based extractors
+    * Improve extraction
+    * Fix 3sat and phoenix
+* [stretchinternet] Fix extraction (#28297)
+* [urplay] Fix episode data extraction (#28292)
++ [bandaichannel] Add support for b-ch.com (#21404)
+* [srgssr] Improve extraction (#14717, #14725, #27231, #28238)
+    + Extract subtitle
+    * Fix extraction for new videos
+    * Update srf download domains
+* [vvvvid] Reduce season request payload size
++ [vvvvid] Extract series sublists playlist title (#27601, #27618)
++ [dplay] Extract Ad-Free uplynk URLs (#28160)
++ [wat] Detect DRM protected videos (#27958)
+* [tf1] Improve extraction (#27980, #28040)
+* [tmz] Fix and improve extraction (#24603, #24687, 28211)
++ [gedidigital] Add support for Gedi group sites (#7347, #26946)
+* [youtube] Fix get_video_info request
+
+
 version 2021.02.22
 
 Core

From 7c06216abff092d43e47b584699d435c40a8115e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 2 Mar 2021 06:19:42 +0700
Subject: [PATCH 251/860] release 2021.03.02

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 docs/supportedsites.md                           | 2 ++
 youtube_dl/version.py                            | 2 +-
 8 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 60879f0ac..9544eaa6c 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.22. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.02. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.02.22**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.02**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.02.22
+ [debug] youtube-dl version 2021.03.02
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index b38d39ab4..c32ebdf56 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.22. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.02. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.02.22**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.02**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 3235de44b..2b5e0f08f 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.22. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.02. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.02.22**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.02**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index a3255623a..13a54982e 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.22. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.02. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.02.22**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.02**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.02.22
+ [debug] youtube-dl version 2021.03.02
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index 124b020c3..dbca582ee 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.02.22. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.02. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.02.22**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.02**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index 07f26f2cf..fbf97a582 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.03.02
 
 Extractors
 * [zdf] Rework extractors (#11606, #13473, #17354, #21185, #26711, #27068,
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 2452c1f7f..2c00ec406 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -82,6 +82,7 @@
  - **awaan:video**
  - **AZMedien**: AZ Medien videos
  - **BaiduVideo**: 百度视频
+ - **bandaichannel**
  - **Bandcamp**
  - **Bandcamp:album**
  - **Bandcamp:weekly**
@@ -330,6 +331,7 @@
  - **Gaskrank**
  - **Gazeta**
  - **GDCVault**
+ - **GediDigital**
  - **generic**: Generic downloader that works on some sites
  - **Gfycat**
  - **GiantBomb**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index f89530293..bfe98aa9f 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.02.22'
+__version__ = '2021.03.02'

From e465b25c1fb0e72b97a032220399d4a959662095 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 24 Feb 2021 11:52:30 +0000
Subject: [PATCH 252/860] [bbc] add support for BBC Reel videos(closes #21870,
 closes #23660, closes #28268)

---
 youtube_dl/extractor/bbc.py | 59 ++++++++++++++++++++++++++++++++-----
 1 file changed, 52 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index b4daee54e..a0c557929 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -5,10 +5,15 @@ import itertools
 import re
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_etree_Element,
+    compat_HTTPError,
+    compat_urlparse,
+)
 from ..utils import (
+    ExtractorError,
     clean_html,
     dict_get,
-    ExtractorError,
     float_or_none,
     get_element_by_class,
     int_or_none,
@@ -21,11 +26,6 @@ from ..utils import (
     urlencode_postdata,
     urljoin,
 )
-from ..compat import (
-    compat_etree_Element,
-    compat_HTTPError,
-    compat_urlparse,
-)
 
 
 class BBCCoUkIE(InfoExtractor):
@@ -793,6 +793,20 @@ class BBCIE(BBCCoUkIE):
             'description': 'Learn English words and phrases from this story',
         },
         'add_ie': [BBCCoUkIE.ie_key()],
+    }, {
+        # BBC Reel
+        'url': 'https://www.bbc.com/reel/video/p07c6sb6/how-positive-thinking-is-harming-your-happiness',
+        'info_dict': {
+            'id': 'p07c6sb9',
+            'ext': 'mp4',
+            'title': 'How positive thinking is harming your happiness',
+            'alt_title': 'The downsides of positive thinking',
+            'description': 'md5:fad74b31da60d83b8265954ee42d85b4',
+            'duration': 235,
+            'thumbnail': r're:https?://.+/p07c9dsr.jpg',
+            'upload_date': '20190604',
+            'categories': ['Psychology'],
+        },
     }]
 
     @classmethod
@@ -980,6 +994,37 @@ class BBCIE(BBCCoUkIE):
                 'subtitles': subtitles,
             }
 
+        # bbc reel (e.g. https://www.bbc.com/reel/video/p07c6sb6/how-positive-thinking-is-harming-your-happiness)
+        initial_data = self._parse_json(self._html_search_regex(
+            r'<script[^>]+id=(["\'])initial-data\1[^>]+data-json=(["\'])(?P<json>(?:(?!\2).)+)',
+            webpage, 'initial data', default='{}', group='json'), playlist_id, fatal=False)
+        if initial_data:
+            init_data = try_get(
+                initial_data, lambda x: x['initData']['items'][0], dict) or {}
+            smp_data = init_data.get('smpData') or {}
+            clip_data = try_get(smp_data, lambda x: x['items'][0], dict) or {}
+            version_id = clip_data.get('versionID')
+            if version_id:
+                title = smp_data['title']
+                formats, subtitles = self._download_media_selector(version_id)
+                self._sort_formats(formats)
+                image_url = smp_data.get('holdingImageURL')
+                display_date = init_data.get('displayDate')
+                topic_title = init_data.get('topicTitle')
+
+                return {
+                    'id': version_id,
+                    'title': title,
+                    'formats': formats,
+                    'alt_title': init_data.get('shortTitle'),
+                    'thumbnail': image_url.replace('$recipe', 'raw') if image_url else None,
+                    'description': smp_data.get('summary') or init_data.get('shortSummary'),
+                    'upload_date': display_date.replace('-', '') if display_date else None,
+                    'subtitles': subtitles,
+                    'duration': int_or_none(clip_data.get('duration')),
+                    'categories': [topic_title] if topic_title else None,
+                }
+
         # Morph based embed (e.g. http://www.bbc.co.uk/sport/live/olympics/36895975)
         # There are several setPayload calls may be present but the video
         # seems to be always related to the first one
@@ -1041,7 +1086,7 @@ class BBCIE(BBCCoUkIE):
                 thumbnail = None
                 image_url = current_programme.get('image_url')
                 if image_url:
-                    thumbnail = image_url.replace('{recipe}', '1920x1920')
+                    thumbnail = image_url.replace('{recipe}', 'raw')
                 return {
                     'id': programme_id,
                     'title': title,

From e1adb3ed4fc911a8177280fe87109e7b54a52fa2 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 2 Mar 2021 11:21:49 +0100
Subject: [PATCH 253/860] [bbc] correct catched exception type

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index a0c557929..92e6f1bea 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -943,7 +943,7 @@ class BBCIE(BBCCoUkIE):
                                     else:
                                         entry['title'] = info['title']
                                         entry['formats'].extend(info['formats'])
-                                except Exception as e:
+                                except ExtractorError as e:
                                     # Some playlist URL may fail with 500, at the same time
                                     # the other one may work fine (e.g.
                                     # http://www.bbc.com/turkce/haberler/2015/06/150615_telabyad_kentin_cogu)

From 8f56907afa693290a6b2e05fb7ffc2f15dca33e2 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 2 Mar 2021 12:04:31 +0100
Subject: [PATCH 254/860] [9c9media] fix extraction for videos with multiple
 ContentPackages(closes #28309)

---
 youtube_dl/extractor/ninecninemedia.py |  4 +---
 youtube_dl/extractor/rds.py            | 18 +++++++++---------
 2 files changed, 10 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/ninecninemedia.py b/youtube_dl/extractor/ninecninemedia.py
index a569c889e..cfc220314 100644
--- a/youtube_dl/extractor/ninecninemedia.py
+++ b/youtube_dl/extractor/ninecninemedia.py
@@ -23,11 +23,9 @@ class NineCNineMediaIE(InfoExtractor):
         destination_code, content_id = re.match(self._VALID_URL, url).groups()
         api_base_url = self._API_BASE_TEMPLATE % (destination_code, content_id)
         content = self._download_json(api_base_url, content_id, query={
-            '$include': '[Media,Season,ContentPackages]',
+            '$include': '[Media.Name,Season,ContentPackages.Duration,ContentPackages.Id]',
         })
         title = content['Name']
-        if len(content['ContentPackages']) > 1:
-            raise ExtractorError('multiple content packages')
         content_package = content['ContentPackages'][0]
         package_id = content_package['Id']
         content_package_url = api_base_url + 'contentpackages/%s/' % package_id
diff --git a/youtube_dl/extractor/rds.py b/youtube_dl/extractor/rds.py
index 8c016a77d..0c497856e 100644
--- a/youtube_dl/extractor/rds.py
+++ b/youtube_dl/extractor/rds.py
@@ -15,17 +15,17 @@ class RDSIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?rds\.ca/vid(?:[eé]|%C3%A9)os/(?:[^/]+/)*(?P<id>[^/]+)-\d+\.\d+'
 
     _TESTS = [{
-        'url': 'http://www.rds.ca/videos/football/nfl/fowler-jr-prend-la-direction-de-jacksonville-3.1132799',
+        # has two 9c9media ContentPackages, the web player selects the first ContentPackage
+        'url': 'https://www.rds.ca/videos/Hockey/NationalHockeyLeague/teams/9/forum-du-5-a-7-jesperi-kotkaniemi-de-retour-de-finlande-3.1377606',
         'info_dict': {
-            'id': '604333',
-            'display_id': 'fowler-jr-prend-la-direction-de-jacksonville',
+            'id': '2083309',
+            'display_id': 'forum-du-5-a-7-jesperi-kotkaniemi-de-retour-de-finlande',
             'ext': 'flv',
-            'title': 'Fowler Jr. prend la direction de Jacksonville',
-            'description': 'Dante Fowler Jr. est le troisième choix du repêchage 2015 de la NFL. ',
-            'timestamp': 1430397346,
-            'upload_date': '20150430',
-            'duration': 154.354,
-            'age_limit': 0,
+            'title': 'Forum du 5 à 7 : Kotkaniemi de retour de Finlande',
+            'description': 'md5:83fa38ecc4a79b19e433433254077f25',
+            'timestamp': 1606129030,
+            'upload_date': '20201123',
+            'duration': 773.039,
         }
     }, {
         'url': 'http://www.rds.ca/vid%C3%A9os/un-voyage-positif-3.877934',

From 061c03013311eff75ac381cb4060204ce91b2510 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 3 Mar 2021 11:42:59 +0700
Subject: [PATCH 255/860] [youtube:tab] Switch continuation to browse API
 (closes #28289, closes #28327)

Until further investigation.
---
 youtube_dl/extractor/youtube.py | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2496d27f1..eb5a58807 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2478,24 +2478,37 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
         headers = {
             'x-youtube-client-name': '1',
             'x-youtube-client-version': '2.20201112.04.01',
+            'content-type': 'application/json',
         }
         if identity_token:
             headers['x-youtube-identity-token'] = identity_token
 
+        data = {
+            'context': {
+                'client': {
+                    'clientName': 'WEB',
+                    'clientVersion': '2.20201021.03.00',
+                }
+            },
+        }
+
         for page_num in itertools.count(1):
             if not continuation:
                 break
+            data['continuation'] = continuation['continuation']
+            data['clickTracking'] = {
+                'clickTrackingParams': continuation['itct']
+            }
             count = 0
             retries = 3
             while count <= retries:
                 try:
                     # Downloading page may result in intermittent 5xx HTTP error
                     # that is usually worked around with a retry
-                    browse = self._download_json(
-                        'https://www.youtube.com/browse_ajax', None,
-                        'Downloading page %d%s'
-                        % (page_num, ' (retry #%d)' % count if count else ''),
-                        headers=headers, query=continuation)
+                    response = self._download_json(
+                        'https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
+                        None, 'Downloading page %d%s' % (page_num, ' (retry #%d)' % count if count else ''),
+                        headers=headers, data=json.dumps(data).encode('utf8'))
                     break
                 except ExtractorError as e:
                     if isinstance(e.cause, compat_HTTPError) and e.cause.code in (500, 503):
@@ -2503,9 +2516,6 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                         if count <= retries:
                             continue
                     raise
-            if not browse:
-                break
-            response = try_get(browse, lambda x: x[1]['response'], dict)
             if not response:
                 break
 

From 8c9766f4bf78ca777e8de9d4809584d8e88098ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 3 Mar 2021 11:44:49 +0700
Subject: [PATCH 256/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index fbf97a582..366d322f5 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,11 @@
+version <unreleased>
+
+Extractors
+* [youtube:tab] Switch continuation to browse API (#28289, #28327)
+* [9c9media] Fix extraction for videos with multiple ContentPackages (#28309)
++ [bbc] Add support for BBC Reel videos (#21870, #23660, #28268)
+
+
 version 2021.03.02
 
 Extractors

From f68692b004f1c65f08a9a7d9c2ee4ab2ec255ea6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 3 Mar 2021 11:47:34 +0700
Subject: [PATCH 257/860] release 2021.03.03

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 youtube_dl/version.py                            | 2 +-
 7 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 9544eaa6c..a8eba3214 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.02. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.03.02**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.03**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.03.02
+ [debug] youtube-dl version 2021.03.03
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index c32ebdf56..7d59a9f2d 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.02. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.03.02**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.03**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 2b5e0f08f..523408f03 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.02. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.03.02**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.03**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index 13a54982e..6e9e094e4 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.02. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.03.02**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.03**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.03.02
+ [debug] youtube-dl version 2021.03.03
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index dbca582ee..46af4e420 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.02. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.03.02**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.03**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index 366d322f5..238ca3965 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.03.03
 
 Extractors
 * [youtube:tab] Switch continuation to browse API (#28289, #28327)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index bfe98aa9f..a1c68e384 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.03.02'
+__version__ = '2021.03.03'

From ec64ec9651848e9173ec033a9a27809e4b5063bc Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 4 Mar 2021 12:41:49 +0100
Subject: [PATCH 258/860] [voxmedia] fix volume embed extraction(closes #28338)

---
 youtube_dl/extractor/voxmedia.py | 26 ++++++++++++++++++--------
 1 file changed, 18 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/voxmedia.py b/youtube_dl/extractor/voxmedia.py
index b318e15d4..661208125 100644
--- a/youtube_dl/extractor/voxmedia.py
+++ b/youtube_dl/extractor/voxmedia.py
@@ -7,6 +7,8 @@ from ..compat import compat_urllib_parse_unquote
 from ..utils import (
     ExtractorError,
     int_or_none,
+    try_get,
+    unified_timestamp,
 )
 
 
@@ -19,14 +21,17 @@ class VoxMediaVolumeIE(OnceIE):
 
         setup = self._parse_json(self._search_regex(
             r'setup\s*=\s*({.+});', webpage, 'setup'), video_id)
-        video_data = setup.get('video') or {}
+        player_setup = setup.get('player_setup') or setup
+        video_data = player_setup.get('video') or {}
+        formatted_metadata = video_data.get('formatted_metadata') or {}
         info = {
             'id': video_id,
-            'title': video_data.get('title_short'),
+            'title': player_setup.get('title') or video_data.get('title_short'),
             'description': video_data.get('description_long') or video_data.get('description_short'),
-            'thumbnail': video_data.get('brightcove_thumbnail')
+            'thumbnail': formatted_metadata.get('thumbnail') or video_data.get('brightcove_thumbnail'),
+            'timestamp': unified_timestamp(formatted_metadata.get('video_publish_date')),
         }
-        asset = setup.get('asset') or setup.get('params') or {}
+        asset = try_get(setup, lambda x: x['embed_assets']['chorus'], dict) or {}
 
         formats = []
         hls_url = asset.get('hls_url')
@@ -47,6 +52,7 @@ class VoxMediaVolumeIE(OnceIE):
         if formats:
             self._sort_formats(formats)
             info['formats'] = formats
+            info['duration'] = int_or_none(asset.get('duration'))
             return info
 
         for provider_video_type in ('ooyala', 'youtube', 'brightcove'):
@@ -84,7 +90,7 @@ class VoxMediaIE(InfoExtractor):
     }, {
         # Volume embed, Youtube
         'url': 'http://www.theverge.com/2014/10/21/7025853/google-nexus-6-hands-on-photos-video-android-phablet',
-        'md5': '4c8f4a0937752b437c3ebc0ed24802b5',
+        'md5': 'fd19aa0cf3a0eea515d4fd5c8c0e9d68',
         'info_dict': {
             'id': 'Gy8Md3Eky38',
             'ext': 'mp4',
@@ -93,6 +99,7 @@ class VoxMediaIE(InfoExtractor):
             'uploader_id': 'TheVerge',
             'upload_date': '20141021',
             'uploader': 'The Verge',
+            'timestamp': 1413907200,
         },
         'add_ie': ['Youtube'],
         'skip': 'similar to the previous test',
@@ -100,13 +107,13 @@ class VoxMediaIE(InfoExtractor):
         # Volume embed, Youtube
         'url': 'http://www.vox.com/2016/3/31/11336640/mississippi-lgbt-religious-freedom-bill',
         'info_dict': {
-            'id': 'YCjDnX-Xzhg',
+            'id': '22986359b',
             'ext': 'mp4',
             'title': "Mississippi's laws are so bad that its anti-LGBTQ law isn't needed to allow discrimination",
             'description': 'md5:fc1317922057de31cd74bce91eb1c66c',
-            'uploader_id': 'voxdotcom',
             'upload_date': '20150915',
-            'uploader': 'Vox',
+            'timestamp': 1442332800,
+            'duration': 285,
         },
         'add_ie': ['Youtube'],
         'skip': 'similar to the previous test',
@@ -160,6 +167,9 @@ class VoxMediaIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Post-Post-PC CEO: The Full Code Conference Video of Microsoft\'s Satya Nadella',
             'description': 'The longtime veteran was chosen earlier this year as the software giant\'s third leader in its history.',
+            'timestamp': 1402938000,
+            'upload_date': '20140616',
+            'duration': 4114,
         },
         'add_ie': ['VoxMediaVolume'],
     }]

From b8b622fbebb158db95edb05a8cc248668194b430 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 4 Mar 2021 17:57:16 +0100
Subject: [PATCH 259/860] [trovo] Add Origin header to VOD formats(closes
 #28346)

---
 youtube_dl/extractor/trovo.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/trovo.py b/youtube_dl/extractor/trovo.py
index 43745213d..de0107aa9 100644
--- a/youtube_dl/extractor/trovo.py
+++ b/youtube_dl/extractor/trovo.py
@@ -153,6 +153,7 @@ class TrovoVodIE(TrovoBaseIE):
                 'protocol': 'm3u8_native',
                 'tbr': int_or_none(play_info.get('bitrate')),
                 'url': play_url,
+                'http_headers': {'Origin': 'https://trovo.live'},
             })
         self._sort_formats(formats)
 

From 7f064d50db957d551dccde73ab73318f53ab3b17 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 7 Mar 2021 08:32:37 +0100
Subject: [PATCH 260/860] [cbs] add support for Paramount+ (closes #28342)

---
 youtube_dl/extractor/cbs.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbs.py b/youtube_dl/extractor/cbs.py
index 4a19a73d2..c79e55a75 100644
--- a/youtube_dl/extractor/cbs.py
+++ b/youtube_dl/extractor/cbs.py
@@ -27,7 +27,7 @@ class CBSBaseIE(ThePlatformFeedIE):
 
 
 class CBSIE(CBSBaseIE):
-    _VALID_URL = r'(?:cbs:|https?://(?:www\.)?(?:cbs\.com/shows/[^/]+/video|colbertlateshow\.com/(?:video|podcasts))/)(?P<id>[\w-]+)'
+    _VALID_URL = r'(?:cbs:|https?://(?:www\.)?(?:(?:cbs|paramountplus)\.com/shows/[^/]+/video|colbertlateshow\.com/(?:video|podcasts))/)(?P<id>[\w-]+)'
 
     _TESTS = [{
         'url': 'http://www.cbs.com/shows/garth-brooks/video/_u7W953k6la293J7EPTd9oHkSPs6Xn6_/connect-chat-feat-garth-brooks/',
@@ -52,6 +52,9 @@ class CBSIE(CBSBaseIE):
     }, {
         'url': 'http://www.colbertlateshow.com/podcasts/dYSwjqPs_X1tvbV_P2FcPWRa_qT6akTC/in-the-bad-room-with-stephen/',
         'only_matching': True,
+    }, {
+        'url': 'https://www.paramountplus.com/shows/all-rise/video/QmR1WhNkh1a_IrdHZrbcRklm176X_rVc/all-rise-space/',
+        'only_matching': True,
     }]
 
     def _extract_video_info(self, content_id, site='cbs', mpx_acc=2198311517):

From c6a14755bb9629967fb12536ee8660ca67ff4345 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 8 Mar 2021 16:53:50 +0100
Subject: [PATCH 261/860] [bilibili] fix video info extraction(closes #28341)

---
 youtube_dl/extractor/bilibili.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 4dc597e16..589fdc1ce 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -156,6 +156,7 @@ class BiliBiliIE(InfoExtractor):
             cid = js['result']['cid']
 
         headers = {
+            'Accept': 'application/json',
             'Referer': url
         }
         headers.update(self.geo_verification_headers())

From 7dc513487fb0babb5257fa72df87c3f24967f2a1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Mar 2021 02:54:10 +0700
Subject: [PATCH 262/860] [pornhub] Extract formats from get_media end point
 (#28395)

---
 youtube_dl/extractor/pornhub.py | 40 +++++++++++++++++++++++----------
 1 file changed, 28 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index b7631e4e1..fdf8b1b0d 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -394,6 +394,21 @@ class PornHubIE(PornHubBaseIE):
 
         upload_date = None
         formats = []
+
+        def add_format(format_url, height=None):
+            tbr = None
+            mobj = re.search(r'(?P<height>\d+)[pP]?_(?P<tbr>\d+)[kK]', format_url)
+            if mobj:
+                if not height:
+                    height = int(mobj.group('height'))
+                tbr = int(mobj.group('tbr'))
+            formats.append({
+                'url': format_url,
+                'format_id': '%dp' % height if height else None,
+                'height': height,
+                'tbr': tbr,
+            })
+
         for video_url, height in video_urls:
             if not upload_date:
                 upload_date = self._search_regex(
@@ -410,18 +425,19 @@ class PornHubIE(PornHubBaseIE):
                     video_url, video_id, 'mp4', entry_protocol='m3u8_native',
                     m3u8_id='hls', fatal=False))
                 continue
-            tbr = None
-            mobj = re.search(r'(?P<height>\d+)[pP]?_(?P<tbr>\d+)[kK]', video_url)
-            if mobj:
-                if not height:
-                    height = int(mobj.group('height'))
-                tbr = int(mobj.group('tbr'))
-            formats.append({
-                'url': video_url,
-                'format_id': '%dp' % height if height else None,
-                'height': height,
-                'tbr': tbr,
-            })
+            if '/video/get_media' in video_url:
+                medias = self._download_json(video_url, video_id, fatal=False)
+                if isinstance(medias, list):
+                    for media in medias:
+                        if not isinstance(media, dict):
+                            continue
+                        video_url = url_or_none(media.get('videoUrl'))
+                        if not video_url:
+                            continue
+                        height = int_or_none(media.get('quality'))
+                        add_format(video_url, height)
+                continue
+            add_format(video_url)
         self._sort_formats(formats)
 
         video_uploader = self._html_search_regex(

From 1a1ccd9a6e8e9e90ab129e89c2524ab3eb9ed2ae Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Mar 2021 02:56:01 +0700
Subject: [PATCH 263/860] [pornhub] Detect flagged videos

---
 youtube_dl/extractor/pornhub.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index fdf8b1b0d..2a7818e41 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -167,6 +167,7 @@ class PornHubIE(PornHubBaseIE):
         'params': {
             'skip_download': True,
         },
+        'skip': 'Video has been flagged for verification in accordance with our trust and safety policy',
     }, {
         # subtitles
         'url': 'https://www.pornhub.com/view_video.php?viewkey=ph5af5fef7c2aa7',
@@ -265,7 +266,8 @@ class PornHubIE(PornHubBaseIE):
         webpage = dl_webpage('pc')
 
         error_msg = self._html_search_regex(
-            r'(?s)<div[^>]+class=(["\'])(?:(?!\1).)*\b(?:removed|userMessageSection)\b(?:(?!\1).)*\1[^>]*>(?P<error>.+?)</div>',
+            (r'(?s)<div[^>]+class=(["\'])(?:(?!\1).)*\b(?:removed|userMessageSection)\b(?:(?!\1).)*\1[^>]*>(?P<error>.+?)</div>',
+             r'(?s)<section[^>]+class=["\']noVideo["\'][^>]*>(?P<error>.+?)</section>'),
             webpage, 'error message', default=None, group='error')
         if error_msg:
             error_msg = re.sub(r'\s+', ' ', error_msg)

From 477bff69065872fff6bab5c3a1b0512018fbb6eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Mar 2021 03:36:31 +0700
Subject: [PATCH 264/860] Introduce release_timestamp meta field (refs #28386)

---
 youtube_dl/YoutubeDL.py        | 20 ++++++++++++--------
 youtube_dl/extractor/common.py |  4 +++-
 2 files changed, 15 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index ecac31f7a..8f65c6499 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1511,14 +1511,18 @@ class YoutubeDL(object):
         if 'display_id' not in info_dict and 'id' in info_dict:
             info_dict['display_id'] = info_dict['id']
 
-        if info_dict.get('upload_date') is None and info_dict.get('timestamp') is not None:
-            # Working around out-of-range timestamp values (e.g. negative ones on Windows,
-            # see http://bugs.python.org/issue1646728)
-            try:
-                upload_date = datetime.datetime.utcfromtimestamp(info_dict['timestamp'])
-                info_dict['upload_date'] = upload_date.strftime('%Y%m%d')
-            except (ValueError, OverflowError, OSError):
-                pass
+        for ts_key, date_key in (
+                ('timestamp', 'upload_date'),
+                ('release_timestamp', 'release_date'),
+        ):
+            if info_dict.get(date_key) is None and info_dict.get(ts_key) is not None:
+                # Working around out-of-range timestamp values (e.g. negative ones on Windows,
+                # see http://bugs.python.org/issue1646728)
+                try:
+                    upload_date = datetime.datetime.utcfromtimestamp(info_dict[ts_key])
+                    info_dict[date_key] = upload_date.strftime('%Y%m%d')
+                except (ValueError, OverflowError, OSError):
+                    pass
 
         # Auto generate title fields corresponding to the *_number fields when missing
         # in order to always have clean titles. This is very common for TV series.
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8eb110f4e..d3b6724df 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -230,8 +230,10 @@ class InfoExtractor(object):
     uploader:       Full name of the video uploader.
     license:        License name the video is licensed under.
     creator:        The creator of the video.
+    release_timestamp: UNIX timestamp of the moment the video was released.
     release_date:   The date (YYYYMMDD) when the video was released.
-    timestamp:      UNIX timestamp of the moment the video became available.
+    timestamp:      UNIX timestamp of the moment the video became available
+                    (uploaded).
     upload_date:    Video upload date (YYYYMMDD).
                     If not explicitly set, calculated from timestamp.
     uploader_id:    Nickname or id of the video uploader.

From 15c24b0346e3951b43dbf29631bfe65292f53ac5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Mar 2021 03:40:56 +0700
Subject: [PATCH 265/860] [lbry] Extract release_timestamp (closes #28386)

---
 youtube_dl/extractor/lbry.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/lbry.py b/youtube_dl/extractor/lbry.py
index 413215a99..95782366b 100644
--- a/youtube_dl/extractor/lbry.py
+++ b/youtube_dl/extractor/lbry.py
@@ -60,6 +60,7 @@ class LBRYBaseIE(InfoExtractor):
             'description': stream_value.get('description'),
             'license': stream_value.get('license'),
             'timestamp': int_or_none(stream.get('timestamp')),
+            'release_timestamp': int_or_none(stream_value.get('release_time')),
             'tags': stream_value.get('tags'),
             'duration': int_or_none(media.get('duration')),
             'channel': try_get(signing_channel, lambda x: x['value']['title']),
@@ -92,6 +93,8 @@ class LBRYIE(LBRYBaseIE):
             'description': 'md5:f6cb5c704b332d37f5119313c2c98f51',
             'timestamp': 1595694354,
             'upload_date': '20200725',
+            'release_timestamp': 1595340697,
+            'release_date': '20200721',
             'width': 1280,
             'height': 720,
         }
@@ -106,6 +109,8 @@ class LBRYIE(LBRYBaseIE):
             'description': 'md5:661ac4f1db09f31728931d7b88807a61',
             'timestamp': 1591312601,
             'upload_date': '20200604',
+            'release_timestamp': 1591312421,
+            'release_date': '20200604',
             'tags': list,
             'duration': 2570,
             'channel': 'The LBRY Foundation',

From bae7dbf78be3a03d8454d1b17bfdbf1bfa0de715 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 10 Mar 2021 03:41:21 +0700
Subject: [PATCH 266/860] [bandcamp] Extract release_timestamp

---
 youtube_dl/extractor/bandcamp.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bandcamp.py b/youtube_dl/extractor/bandcamp.py
index 69e673a26..006aab3b4 100644
--- a/youtube_dl/extractor/bandcamp.py
+++ b/youtube_dl/extractor/bandcamp.py
@@ -49,6 +49,7 @@ class BandcampIE(InfoExtractor):
             'uploader': 'Ben Prunty',
             'timestamp': 1396508491,
             'upload_date': '20140403',
+            'release_timestamp': 1396483200,
             'release_date': '20140403',
             'duration': 260.877,
             'track': 'Lanius (Battle)',
@@ -69,6 +70,7 @@ class BandcampIE(InfoExtractor):
             'uploader': 'Mastodon',
             'timestamp': 1322005399,
             'upload_date': '20111122',
+            'release_timestamp': 1076112000,
             'release_date': '20040207',
             'duration': 120.79,
             'track': 'Hail to Fire',
@@ -197,7 +199,7 @@ class BandcampIE(InfoExtractor):
             'thumbnail': thumbnail,
             'uploader': artist,
             'timestamp': timestamp,
-            'release_date': unified_strdate(tralbum.get('album_release_date')),
+            'release_timestamp': unified_timestamp(tralbum.get('album_release_date')),
             'duration': duration,
             'track': track,
             'track_number': track_number,

From 64ed3af328929f83b02ed57df5cb4f863fdd0389 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 10 Mar 2021 11:45:30 +0100
Subject: [PATCH 267/860] [lbry] add support for channel filters(closes #28385)

---
 youtube_dl/extractor/lbry.py | 46 ++++++++++++++++++++++++++++--------
 1 file changed, 36 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/lbry.py b/youtube_dl/extractor/lbry.py
index 95782366b..ae43d56ea 100644
--- a/youtube_dl/extractor/lbry.py
+++ b/youtube_dl/extractor/lbry.py
@@ -6,8 +6,10 @@ import json
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_parse_qs,
     compat_str,
     compat_urllib_parse_unquote,
+    compat_urllib_parse_urlparse,
 )
 from ..utils import (
     determine_ext,
@@ -186,17 +188,18 @@ class LBRYChannelIE(LBRYBaseIE):
     }]
     _PAGE_SIZE = 50
 
-    def _fetch_page(self, claim_id, url, page):
+    def _fetch_page(self, claim_id, url, params, page):
         page += 1
+        page_params = {
+            'channel_ids': [claim_id],
+            'claim_type': 'stream',
+            'no_totals': True,
+            'page': page,
+            'page_size': self._PAGE_SIZE,
+        }
+        page_params.update(params)
         result = self._call_api_proxy(
-            'claim_search', claim_id, {
-                'channel_ids': [claim_id],
-                'claim_type': 'stream',
-                'no_totals': True,
-                'page': page,
-                'page_size': self._PAGE_SIZE,
-                'stream_types': self._SUPPORTED_STREAM_TYPES,
-            }, 'page %d' % page)
+            'claim_search', claim_id, page_params, 'page %d' % page)
         for item in (result.get('items') or []):
             stream_claim_name = item.get('name')
             stream_claim_id = item.get('claim_id')
@@ -217,8 +220,31 @@ class LBRYChannelIE(LBRYBaseIE):
         result = self._resolve_url(
             'lbry://' + display_id, display_id, 'channel')
         claim_id = result['claim_id']
+        qs = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+        content = qs.get('content', [None])[0]
+        params = {
+            'fee_amount': qs.get('fee_amount', ['>=0'])[0],
+            'order_by': {
+                'new': ['release_time'],
+                'top': ['effective_amount'],
+                'trending': ['trending_group', 'trending_mixed'],
+            }[qs.get('order', ['new'])[0]],
+            'stream_types': [content] if content in ['audio', 'video'] else self._SUPPORTED_STREAM_TYPES,
+        }
+        duration = qs.get('duration', [None])[0]
+        if duration:
+            params['duration'] = {
+                'long': '>=1200',
+                'short': '<=240',
+            }[duration]
+        language = qs.get('language', ['all'])[0]
+        if language != 'all':
+            languages = [language]
+            if language == 'en':
+                languages.append('none')
+            params['any_languages'] = languages
         entries = OnDemandPagedList(
-            functools.partial(self._fetch_page, claim_id, url),
+            functools.partial(self._fetch_page, claim_id, url, params),
             self._PAGE_SIZE)
         result_value = result.get('value') or {}
         return self.playlist_result(

From fc2c6d53239d4b4a6bac5383441152117ccf3c6f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 10 Mar 2021 13:16:21 +0100
Subject: [PATCH 268/860] [shahid] fix format extraction(closes #28383)

---
 youtube_dl/extractor/shahid.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index 5c2a6206b..b5e093bd2 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -51,13 +51,16 @@ class ShahidIE(ShahidBaseIE):
     _NETRC_MACHINE = 'shahid'
     _VALID_URL = r'https?://shahid\.mbc\.net/ar/(?:serie|show|movie)s/[^/]+/(?P<type>episode|clip|movie)-(?P<id>\d+)'
     _TESTS = [{
-        'url': 'https://shahid.mbc.net/ar/shows/%D9%85%D8%AC%D9%84%D8%B3-%D8%A7%D9%84%D8%B4%D8%A8%D8%A7%D8%A8-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D9%83%D9%84%D9%8A%D8%A8-1/clip-275286',
+        'url': 'https://shahid.mbc.net/ar/shows/%D9%85%D8%AA%D8%AD%D9%81-%D8%A7%D9%84%D8%AF%D8%AD%D9%8A%D8%AD-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D9%83%D9%84%D9%8A%D8%A8-1/clip-816924',
         'info_dict': {
-            'id': '275286',
+            'id': '816924',
             'ext': 'mp4',
-            'title': 'مجلس الشباب الموسم 1 كليب 1',
-            'timestamp': 1506988800,
-            'upload_date': '20171003',
+            'title': 'متحف الدحيح الموسم 1 كليب 1',
+            'timestamp': 1602806400,
+            'upload_date': '20201016',
+            'description': 'برومو',
+            'duration': 22,
+            'categories': ['كوميديا'],
         },
         'params': {
             # m3u8 download
@@ -109,12 +112,15 @@ class ShahidIE(ShahidBaseIE):
             page_type = 'episode'
 
         playout = self._call_api(
-            'playout/url/' + video_id, video_id)['playout']
+            'playout/new/url/' + video_id, video_id)['playout']
 
         if playout.get('drm'):
             raise ExtractorError('This video is DRM protected.', expected=True)
 
-        formats = self._extract_m3u8_formats(playout['url'], video_id, 'mp4')
+        formats = self._extract_m3u8_formats(re.sub(
+            # https://docs.aws.amazon.com/mediapackage/latest/ug/manifest-filtering.html
+            r'aws\.manifestfilter=[\w:;,-]+&?',
+            '', playout['url']), video_id, 'mp4')
         self._sort_formats(formats)
 
         # video = self._call_api(

From 9c644a641922e5ac3b5b4a1c9386fa599973e885 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 12 Mar 2021 09:51:01 +0100
Subject: [PATCH 269/860] [fujitv] fix HLS formats extension(closes #28416)

---
 youtube_dl/extractor/fujitv.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/fujitv.py b/youtube_dl/extractor/fujitv.py
index 39685e075..a02a94374 100644
--- a/youtube_dl/extractor/fujitv.py
+++ b/youtube_dl/extractor/fujitv.py
@@ -17,7 +17,7 @@ class FujiTVFODPlus7IE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         formats = self._extract_m3u8_formats(
-            self._BASE_URL + 'abr/pc_html5/%s.m3u8' % video_id, video_id)
+            self._BASE_URL + 'abr/pc_html5/%s.m3u8' % video_id, video_id, 'mp4')
         for f in formats:
             wh = self._BITRATE_MAP.get(f.get('tbr'))
             if wh:

From 43d986acd8bf7247725fc9de34648c0eda560daf Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 12 Mar 2021 10:14:28 +0100
Subject: [PATCH 270/860] [tver] improve title extraction(closes #28418)

---
 youtube_dl/extractor/tver.py | 11 ++++++++++-
 1 file changed, 10 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/tver.py b/youtube_dl/extractor/tver.py
index 931d4d650..a54f49319 100644
--- a/youtube_dl/extractor/tver.py
+++ b/youtube_dl/extractor/tver.py
@@ -9,6 +9,7 @@ from ..utils import (
     int_or_none,
     remove_start,
     smuggle_url,
+    strip_or_none,
     try_get,
 )
 
@@ -25,6 +26,10 @@ class TVerIE(InfoExtractor):
     }, {
         'url': 'https://tver.jp/episode/79622438',
         'only_matching': True,
+    }, {
+        # subtitle = ' '
+        'url': 'https://tver.jp/corner/f0068870',
+        'only_matching': True,
     }]
     _TOKEN = None
     BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/default_default/index.html?videoId=%s'
@@ -47,8 +52,12 @@ class TVerIE(InfoExtractor):
         }
 
         if service == 'cx':
+            title = main['title']
+            subtitle = strip_or_none(main.get('subtitle'))
+            if subtitle:
+                title += ' - ' + subtitle
             info.update({
-                'title': main.get('subtitle') or main['title'],
+                'title': title,
                 'url': 'https://i.fod.fujitv.co.jp/plus7/web/%s/%s.html' % (p_id[:4], p_id),
                 'ie_key': 'FujiTVFODPlus7',
             })

From ef414343e5fa2bc4fddae3097ecde5a8e32c2d4c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 12 Mar 2021 10:48:58 +0100
Subject: [PATCH 271/860] [peertube] improve thumbnail extraction(closes
 #28419)

---
 youtube_dl/extractor/peertube.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/peertube.py b/youtube_dl/extractor/peertube.py
index 32ff51653..d9b13adc2 100644
--- a/youtube_dl/extractor/peertube.py
+++ b/youtube_dl/extractor/peertube.py
@@ -599,11 +599,13 @@ class PeerTubeIE(InfoExtractor):
         else:
             age_limit = None
 
+        webpage_url = 'https://%s/videos/watch/%s' % (host, video_id)
+
         return {
             'id': video_id,
             'title': title,
             'description': description,
-            'thumbnail': urljoin(url, video.get('thumbnailPath')),
+            'thumbnail': urljoin(webpage_url, video.get('thumbnailPath')),
             'timestamp': unified_timestamp(video.get('publishedAt')),
             'uploader': account_data('displayName', compat_str),
             'uploader_id': str_or_none(account_data('id', int)),
@@ -621,5 +623,6 @@ class PeerTubeIE(InfoExtractor):
             'tags': try_get(video, lambda x: x['tags'], list),
             'categories': categories,
             'formats': formats,
-            'subtitles': subtitles
+            'subtitles': subtitles,
+            'webpage_url': webpage_url,
         }

From 1182f9567b86f2af747cdb8769ab87649c8ce4c2 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 12 Mar 2021 18:11:11 +0100
Subject: [PATCH 272/860] [pinterest] reduce the number of HLS format requests

---
 youtube_dl/extractor/pinterest.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pinterest.py b/youtube_dl/extractor/pinterest.py
index b249c9eda..42528d746 100644
--- a/youtube_dl/extractor/pinterest.py
+++ b/youtube_dl/extractor/pinterest.py
@@ -31,6 +31,7 @@ class PinterestBaseIE(InfoExtractor):
 
         title = (data.get('title') or data.get('grid_title') or video_id).strip()
 
+        urls = []
         formats = []
         duration = None
         if extract_formats:
@@ -38,8 +39,9 @@ class PinterestBaseIE(InfoExtractor):
                 if not isinstance(format_dict, dict):
                     continue
                 format_url = url_or_none(format_dict.get('url'))
-                if not format_url:
+                if not format_url or format_url in urls:
                     continue
+                urls.append(format_url)
                 duration = float_or_none(format_dict.get('duration'), scale=1000)
                 ext = determine_ext(format_url)
                 if 'hls' in format_id.lower() or ext == 'm3u8':

From 60845121ca2f49172e7cd941c0cb43363cb86e46 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 13 Mar 2021 15:19:24 +0100
Subject: [PATCH 273/860] [sportdeutschland] fix extraction(closes
 #21856)(closes #28425)

---
 youtube_dl/extractor/sportdeutschland.py | 147 +++++++++++++----------
 1 file changed, 85 insertions(+), 62 deletions(-)

diff --git a/youtube_dl/extractor/sportdeutschland.py b/youtube_dl/extractor/sportdeutschland.py
index 378fc7568..3e497a939 100644
--- a/youtube_dl/extractor/sportdeutschland.py
+++ b/youtube_dl/extractor/sportdeutschland.py
@@ -1,82 +1,105 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..compat import (
+    compat_parse_qs,
+    compat_urllib_parse_urlparse,
+)
 from ..utils import (
+    clean_html,
+    float_or_none,
+    int_or_none,
     parse_iso8601,
-    sanitized_Request,
+    strip_or_none,
+    try_get,
 )
 
 
 class SportDeutschlandIE(InfoExtractor):
-    _VALID_URL = r'https?://sportdeutschland\.tv/(?P<sport>[^/?#]+)/(?P<id>[^?#/]+)(?:$|[?#])'
+    _VALID_URL = r'https?://sportdeutschland\.tv/(?P<id>(?:[^/]+/)?[^?#/&]+)'
     _TESTS = [{
         'url': 'https://sportdeutschland.tv/badminton/re-live-deutsche-meisterschaften-2020-halbfinals?playlistId=0',
         'info_dict': {
-            'id': 're-live-deutsche-meisterschaften-2020-halbfinals',
+            'id': '5318cac0275701382770543d7edaf0a0',
             'ext': 'mp4',
-            'title': 're:Re-live: Deutsche Meisterschaften 2020.*Halbfinals',
-            'categories': ['Badminton-Deutschland'],
-            'view_count': int,
-            'thumbnail': r're:^https?://.*\.(?:jpg|png)$',
-            'timestamp': int,
-            'upload_date': '20200201',
-            'description': 're:.*',  # meaningless description for THIS video
+            'title': 'Re-live: Deutsche Meisterschaften 2020 - Halbfinals - Teil 1',
+            'duration': 16106.36,
         },
+        'params': {
+            'noplaylist': True,
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        'url': 'https://sportdeutschland.tv/badminton/re-live-deutsche-meisterschaften-2020-halbfinals?playlistId=0',
+        'info_dict': {
+            'id': 'c6e2fdd01f63013854c47054d2ab776f',
+            'title': 'Re-live: Deutsche Meisterschaften 2020 - Halbfinals',
+            'description': 'md5:5263ff4c31c04bb780c9f91130b48530',
+            'duration': 31397,
+        },
+        'playlist_count': 2,
+    }, {
+        'url': 'https://sportdeutschland.tv/freeride-world-tour-2021-fieberbrunn-oesterreich',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        sport_id = mobj.group('sport')
-
-        api_url = 'https://proxy.vidibusdynamic.net/ssl/backend.sportdeutschland.tv/api/permalinks/%s/%s?access_token=true' % (
-            sport_id, video_id)
-        req = sanitized_Request(api_url, headers={
-            'Accept': 'application/vnd.vidibus.v2.html+json',
-            'Referer': url,
-        })
-        data = self._download_json(req, video_id)
-
+        display_id = self._match_id(url)
+        data = self._download_json(
+            'https://backend.sportdeutschland.tv/api/permalinks/' + display_id,
+            display_id, query={'access_token': 'true'})
         asset = data['asset']
-        categories = [data['section']['title']]
-
-        formats = []
-        smil_url = asset['video']
-        if '.smil' in smil_url:
-            m3u8_url = smil_url.replace('.smil', '.m3u8')
-            formats.extend(
-                self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4'))
-
-            smil_doc = self._download_xml(
-                smil_url, video_id, note='Downloading SMIL metadata')
-            base_url_el = smil_doc.find('./head/meta')
-            if base_url_el:
-                base_url = base_url_el.attrib['base']
-            formats.extend([{
-                'format_id': 'rmtp',
-                'url': base_url if base_url_el else n.attrib['src'],
-                'play_path': n.attrib['src'],
-                'ext': 'flv',
-                'preference': -100,
-                'format_note': 'Seems to fail at example stream',
-            } for n in smil_doc.findall('./body/video')])
-        else:
-            formats.append({'url': smil_url})
-
-        self._sort_formats(formats)
-
-        return {
-            'id': video_id,
-            'formats': formats,
-            'title': asset['title'],
-            'thumbnail': asset.get('image'),
-            'description': asset.get('teaser'),
-            'duration': asset.get('duration'),
-            'categories': categories,
-            'view_count': asset.get('views'),
-            'rtmp_live': asset.get('live'),
-            'timestamp': parse_iso8601(asset.get('date')),
+        title = (asset.get('title') or asset['label']).strip()
+        asset_id = asset.get('id') or asset.get('uuid')
+        info = {
+            'id': asset_id,
+            'title': title,
+            'description': clean_html(asset.get('body') or asset.get('description')) or asset.get('teaser'),
+            'duration': int_or_none(asset.get('seconds')),
         }
+        videos = asset.get('videos') or []
+        if len(videos) > 1:
+            playlist_id = compat_parse_qs(compat_urllib_parse_urlparse(url).query).get('playlistId', [None])[0]
+            if playlist_id:
+                if self._downloader.params.get('noplaylist'):
+                    videos = [videos[int(playlist_id)]]
+                    self.to_screen('Downloading just a single video because of --no-playlist')
+                else:
+                    self.to_screen('Downloading playlist %s - add --no-playlist to just download video' % asset_id)
+
+            def entries():
+                for i, video in enumerate(videos, 1):
+                    video_id = video.get('uuid')
+                    video_url = video.get('url')
+                    if not (video_id and video_url):
+                        continue
+                    formats = self._extract_m3u8_formats(
+                        video_url.replace('.smil', '.m3u8'), video_id, 'mp4', fatal=False)
+                    if not formats:
+                        continue
+                    yield {
+                        'id': video_id,
+                        'formats': formats,
+                        'title': title + ' - ' + (video.get('label') or 'Teil %d' % i),
+                        'duration': float_or_none(video.get('duration')),
+                    }
+            info.update({
+                '_type': 'multi_video',
+                'entries': entries(),
+            })
+        else:
+            formats = self._extract_m3u8_formats(
+                videos[0]['url'].replace('.smil', '.m3u8'), asset_id, 'mp4')
+            section_title = strip_or_none(try_get(data, lambda x: x['section']['title']))
+            info.update({
+                'formats': formats,
+                'display_id': asset.get('permalink'),
+                'thumbnail': try_get(asset, lambda x: x['images'][0]),
+                'categories': [section_title] if section_title else None,
+                'view_count': int_or_none(asset.get('views')),
+                'is_live': asset.get('is_live') is True,
+                'timestamp': parse_iso8601(asset.get('date') or asset.get('published_at')),
+            })
+        return info

From 1860d0f41cf50b1a0876174c4e1ee7adbbd4a0f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Mar 2021 09:26:54 +0700
Subject: [PATCH 274/860] [southpark] Fix extraction and add support for
 southparkstudios.com (closes #26763, closes #28413)

---
 youtube_dl/extractor/southpark.py | 16 ++++++++++++++--
 1 file changed, 14 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/southpark.py b/youtube_dl/extractor/southpark.py
index da75a43a7..0774da06e 100644
--- a/youtube_dl/extractor/southpark.py
+++ b/youtube_dl/extractor/southpark.py
@@ -6,9 +6,9 @@ from .mtv import MTVServicesInfoExtractor
 
 class SouthParkIE(MTVServicesInfoExtractor):
     IE_NAME = 'southpark.cc.com'
-    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark\.cc\.com/(?:clips|(?:full-)?episodes|collections)/(?P<id>.+?)(\?|#|$))'
+    _VALID_URL = r'https?://(?:www\.)?(?P<url>southpark(?:\.cc|studios)\.com/(?:clips|(?:full-)?episodes|collections)/(?P<id>.+?)(\?|#|$))'
 
-    _FEED_URL = 'http://www.southparkstudios.com/feeds/video-player/mrss'
+    _FEED_URL = 'http://feeds.mtvnservices.com/od/feed/intl-mrss-player-feed'
 
     _TESTS = [{
         'url': 'http://southpark.cc.com/clips/104437/bat-daded#tab=featured',
@@ -23,8 +23,20 @@ class SouthParkIE(MTVServicesInfoExtractor):
     }, {
         'url': 'http://southpark.cc.com/collections/7758/fan-favorites/1',
         'only_matching': True,
+    }, {
+        'url': 'https://www.southparkstudios.com/episodes/h4o269/south-park-stunning-and-brave-season-19-ep-1',
+        'only_matching': True,
     }]
 
+    def _get_feed_query(self, uri):
+        return {
+            'accountOverride': 'intl.mtvi.com',
+            'arcEp': 'shared.southpark.global',
+            'ep': '90877963',
+            'imageEp': 'shared.southpark.global',
+            'mgid': uri,
+        }
+
 
 class SouthParkEsIE(SouthParkIE):
     IE_NAME = 'southpark.cc.com:español'

From b509d24b2fef8d5994b7d925db51befcbbf996fb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Mar 2021 09:36:11 +0700
Subject: [PATCH 275/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 238ca3965..924d202b0 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,28 @@
+version <unreleased>
+
+Core
++ Introduce release_timestamp meta field (#28386)
+
+Extractors
++ [southpark] Add support for southparkstudios.com (#28413)
+* [southpark] Fix extraction (#26763, #28413)
+* [sportdeutschland] Fix extraction (#21856, #28425)
+* [pinterest] Reduce the number of HLS format requests
+* [peertube] Improve thumbnail extraction (#28419)
+* [tver] Improve title extraction (#28418)
+* [fujitv] Fix HLS formats extension (#28416)
+* [shahid] Fix format extraction (#28383)
++ [lbry] Add support for channel filters (#28385)
++ [bandcamp] Extract release timestamp
++ [lbry] Extract release timestamp (#28386)
+* [pornhub] Detect flagged videos
++ [pornhub] Extract formats from get_media end point (#28395)
+* [bilibili] Fix video info extraction (#28341)
++ [cbs] Add support for Paramount+ (#28342)
++ [trovo] Add Origin header to VOD formats (#28346)
+* [voxmedia] Fix volume embed extraction (#28338)
+
+
 version 2021.03.03
 
 Extractors

From ebfd66c4b1d6ffabd8a5bc52737f2bacac341d3d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 14 Mar 2021 09:38:16 +0700
Subject: [PATCH 276/860] release 2021.03.14

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 youtube_dl/version.py                            | 2 +-
 7 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index a8eba3214..9cc79eff4 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.14. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.03.03**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.14**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.03.03
+ [debug] youtube-dl version 2021.03.14
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index 7d59a9f2d..3296e44a8 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.14. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.03.03**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.14**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 523408f03..f74c29736 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.14. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.03.03**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.14**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index 6e9e094e4..ae9e273ef 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.14. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.03.03**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.14**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.03.03
+ [debug] youtube-dl version 2021.03.14
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index 46af4e420..04fbea2f6 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.03. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.14. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.03.03**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.14**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index 924d202b0..73fe316b9 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.03.14
 
 Core
 + Introduce release_timestamp meta field (#28386)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a1c68e384..5a540119c 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.03.03'
+__version__ = '2021.03.14'

From 9955bb4a2704f98b74a448c82dfd690ec6775b8d Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 14 Mar 2021 14:49:23 +0100
Subject: [PATCH 277/860] [rtve] improve extraction

- extract all formats
- fix RTVE Infantil extraction(closes #24851)
- extract is_live and series
---
 youtube_dl/extractor/rtve.py | 232 ++++++++++++++++-------------------
 1 file changed, 104 insertions(+), 128 deletions(-)

diff --git a/youtube_dl/extractor/rtve.py b/youtube_dl/extractor/rtve.py
index ce9db0629..d2fb754cf 100644
--- a/youtube_dl/extractor/rtve.py
+++ b/youtube_dl/extractor/rtve.py
@@ -2,8 +2,9 @@
 from __future__ import unicode_literals
 
 import base64
+import io
 import re
-import time
+import sys
 
 from .common import InfoExtractor
 from ..compat import (
@@ -14,56 +15,13 @@ from ..utils import (
     determine_ext,
     ExtractorError,
     float_or_none,
+    qualities,
     remove_end,
     remove_start,
-    sanitized_Request,
     std_headers,
 )
 
-
-def _decrypt_url(png):
-    encrypted_data = compat_b64decode(png)
-    text_index = encrypted_data.find(b'tEXt')
-    text_chunk = encrypted_data[text_index - 4:]
-    length = compat_struct_unpack('!I', text_chunk[:4])[0]
-    # Use bytearray to get integers when iterating in both python 2.x and 3.x
-    data = bytearray(text_chunk[8:8 + length])
-    data = [chr(b) for b in data if b != 0]
-    hash_index = data.index('#')
-    alphabet_data = data[:hash_index]
-    url_data = data[hash_index + 1:]
-    if url_data[0] == 'H' and url_data[3] == '%':
-        # remove useless HQ%% at the start
-        url_data = url_data[4:]
-
-    alphabet = []
-    e = 0
-    d = 0
-    for l in alphabet_data:
-        if d == 0:
-            alphabet.append(l)
-            d = e = (e + 1) % 4
-        else:
-            d -= 1
-    url = ''
-    f = 0
-    e = 3
-    b = 1
-    for letter in url_data:
-        if f == 0:
-            l = int(letter) * 10
-            f = 1
-        else:
-            if e == 0:
-                l += int(letter)
-                url += alphabet[l]
-                e = (b + 3) % 4
-                f = 0
-                b += 1
-            else:
-                e -= 1
-
-    return url
+_bytes_to_chr = (lambda x: x) if sys.version_info[0] == 2 else (lambda x: map(chr, x))
 
 
 class RTVEALaCartaIE(InfoExtractor):
@@ -79,28 +37,31 @@ class RTVEALaCartaIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Balonmano - Swiss Cup masculina. Final: España-Suecia',
             'duration': 5024.566,
+            'series': 'Balonmano',
         },
+        'expected_warnings': ['Failed to download MPD manifest', 'Failed to download m3u8 information'],
     }, {
         'note': 'Live stream',
         'url': 'http://www.rtve.es/alacarta/videos/television/24h-live/1694255/',
         'info_dict': {
             'id': '1694255',
-            'ext': 'flv',
-            'title': 'TODO',
+            'ext': 'mp4',
+            'title': 're:^24H LIVE [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': 'live stream',
         },
-        'skip': 'The f4m manifest can\'t be used yet',
     }, {
         'url': 'http://www.rtve.es/alacarta/videos/servir-y-proteger/servir-proteger-capitulo-104/4236788/',
-        'md5': 'e55e162379ad587e9640eda4f7353c0f',
+        'md5': 'd850f3c8731ea53952ebab489cf81cbf',
         'info_dict': {
             'id': '4236788',
             'ext': 'mp4',
-            'title': 'Servir y proteger - Capítulo 104 ',
+            'title': 'Servir y proteger - Capítulo 104',
             'duration': 3222.0,
         },
-        'params': {
-            'skip_download': True,  # requires ffmpeg
-        },
+        'expected_warnings': ['Failed to download MPD manifest', 'Failed to download m3u8 information'],
     }, {
         'url': 'http://www.rtve.es/m/alacarta/videos/cuentame-como-paso/cuentame-como-paso-t16-ultimo-minuto-nuestra-vida-capitulo-276/2969138/?media=tve',
         'only_matching': True,
@@ -111,58 +72,102 @@ class RTVEALaCartaIE(InfoExtractor):
 
     def _real_initialize(self):
         user_agent_b64 = base64.b64encode(std_headers['User-Agent'].encode('utf-8')).decode('utf-8')
-        manager_info = self._download_json(
+        self._manager = self._download_json(
             'http://www.rtve.es/odin/loki/' + user_agent_b64,
-            None, 'Fetching manager info')
-        self._manager = manager_info['manager']
+            None, 'Fetching manager info')['manager']
+
+    @staticmethod
+    def _decrypt_url(png):
+        encrypted_data = io.BytesIO(compat_b64decode(png)[8:])
+        while True:
+            length = compat_struct_unpack('!I', encrypted_data.read(4))[0]
+            chunk_type = encrypted_data.read(4)
+            if chunk_type == b'IEND':
+                break
+            data = encrypted_data.read(length)
+            if chunk_type == b'tEXt':
+                alphabet_data, text = data.split(b'\0')
+                quality, url_data = text.split(b'%%')
+                alphabet = []
+                e = 0
+                d = 0
+                for l in _bytes_to_chr(alphabet_data):
+                    if d == 0:
+                        alphabet.append(l)
+                        d = e = (e + 1) % 4
+                    else:
+                        d -= 1
+                url = ''
+                f = 0
+                e = 3
+                b = 1
+                for letter in _bytes_to_chr(url_data):
+                    if f == 0:
+                        l = int(letter) * 10
+                        f = 1
+                    else:
+                        if e == 0:
+                            l += int(letter)
+                            url += alphabet[l]
+                            e = (b + 3) % 4
+                            f = 0
+                            b += 1
+                        else:
+                            e -= 1
+
+                yield quality.decode(), url
+            encrypted_data.read(4)  # CRC
+
+    def _extract_png_formats(self, video_id):
+        png = self._download_webpage(
+            'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id),
+            video_id, 'Downloading url information', query={'q': 'v2'})
+        q = qualities(['Media', 'Alta', 'HQ', 'HD_READY', 'HD_FULL'])
+        formats = []
+        for quality, video_url in self._decrypt_url(png):
+            ext = determine_ext(video_url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    video_url, video_id, 'mp4', 'm3u8_native',
+                    m3u8_id='hls', fatal=False))
+            elif ext == 'mpd':
+                formats.extend(self._extract_mpd_formats(
+                    video_url, video_id, 'dash', fatal=False))
+            else:
+                formats.append({
+                    'format_id': quality,
+                    'quality': q(quality),
+                    'url': video_url,
+                })
+        self._sort_formats(formats)
+        return formats
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
+        video_id = self._match_id(url)
         info = self._download_json(
             'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
             video_id)['page']['items'][0]
         if info['state'] == 'DESPU':
             raise ExtractorError('The video is no longer available', expected=True)
-        title = info['title']
-        png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/%s/videos/%s.png' % (self._manager, video_id)
-        png_request = sanitized_Request(png_url)
-        png_request.add_header('Referer', url)
-        png = self._download_webpage(png_request, video_id, 'Downloading url information')
-        video_url = _decrypt_url(png)
-        ext = determine_ext(video_url)
-
-        formats = []
-        if not video_url.endswith('.f4m') and ext != 'm3u8':
-            if '?' not in video_url:
-                video_url = video_url.replace('resources/', 'auth/resources/')
-            video_url = video_url.replace('.net.rtve', '.multimedia.cdn.rtve')
-
-        if ext == 'm3u8':
-            formats.extend(self._extract_m3u8_formats(
-                video_url, video_id, ext='mp4', entry_protocol='m3u8_native',
-                m3u8_id='hls', fatal=False))
-        elif ext == 'f4m':
-            formats.extend(self._extract_f4m_formats(
-                video_url, video_id, f4m_id='hds', fatal=False))
-        else:
-            formats.append({
-                'url': video_url,
-            })
-        self._sort_formats(formats)
+        title = info['title'].strip()
+        formats = self._extract_png_formats(video_id)
 
         subtitles = None
-        if info.get('sbtFile') is not None:
-            subtitles = self.extract_subtitles(video_id, info['sbtFile'])
+        sbt_file = info.get('sbtFile')
+        if sbt_file:
+            subtitles = self.extract_subtitles(video_id, sbt_file)
+
+        is_live = info.get('live') is True
 
         return {
             'id': video_id,
-            'title': title,
+            'title': self._live_title(title) if is_live else title,
             'formats': formats,
             'thumbnail': info.get('image'),
-            'page_url': url,
             'subtitles': subtitles,
-            'duration': float_or_none(info.get('duration'), scale=1000),
+            'duration': float_or_none(info.get('duration'), 1000),
+            'is_live': is_live,
+            'series': info.get('programTitle'),
         }
 
     def _get_subtitles(self, video_id, sub_file):
@@ -174,48 +179,26 @@ class RTVEALaCartaIE(InfoExtractor):
             for s in subs)
 
 
-class RTVEInfantilIE(InfoExtractor):
+class RTVEInfantilIE(RTVEALaCartaIE):
     IE_NAME = 'rtve.es:infantil'
     IE_DESC = 'RTVE infantil'
-    _VALID_URL = r'https?://(?:www\.)?rtve\.es/infantil/serie/(?P<show>[^/]*)/video/(?P<short_title>[^/]*)/(?P<id>[0-9]+)/'
+    _VALID_URL = r'https?://(?:www\.)?rtve\.es/infantil/serie/[^/]+/video/[^/]+/(?P<id>[0-9]+)/'
 
     _TESTS = [{
         'url': 'http://www.rtve.es/infantil/serie/cleo/video/maneras-vivir/3040283/',
-        'md5': '915319587b33720b8e0357caaa6617e6',
+        'md5': '5747454717aedf9f9fdf212d1bcfc48d',
         'info_dict': {
             'id': '3040283',
             'ext': 'mp4',
             'title': 'Maneras de vivir',
-            'thumbnail': 'http://www.rtve.es/resources/jpg/6/5/1426182947956.JPG',
+            'thumbnail': r're:https?://.+/1426182947956\.JPG',
             'duration': 357.958,
         },
+        'expected_warnings': ['Failed to download MPD manifest', 'Failed to download m3u8 information'],
     }]
 
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        info = self._download_json(
-            'http://www.rtve.es/api/videos/%s/config/alacarta_videos.json' % video_id,
-            video_id)['page']['items'][0]
 
-        webpage = self._download_webpage(url, video_id)
-        vidplayer_id = self._search_regex(
-            r' id="vidplayer([0-9]+)"', webpage, 'internal video ID')
-
-        png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/default/videos/%s.png' % vidplayer_id
-        png = self._download_webpage(png_url, video_id, 'Downloading url information')
-        video_url = _decrypt_url(png)
-
-        return {
-            'id': video_id,
-            'ext': 'mp4',
-            'title': info['title'],
-            'url': video_url,
-            'thumbnail': info.get('image'),
-            'duration': float_or_none(info.get('duration'), scale=1000),
-        }
-
-
-class RTVELiveIE(InfoExtractor):
+class RTVELiveIE(RTVEALaCartaIE):
     IE_NAME = 'rtve.es:live'
     IE_DESC = 'RTVE.es live streams'
     _VALID_URL = r'https?://(?:www\.)?rtve\.es/directo/(?P<id>[a-zA-Z0-9-]+)'
@@ -225,7 +208,7 @@ class RTVELiveIE(InfoExtractor):
         'info_dict': {
             'id': 'la-1',
             'ext': 'mp4',
-            'title': 're:^La 1 [0-9]{4}-[0-9]{2}-[0-9]{2}Z[0-9]{6}$',
+            'title': 're:^La 1 [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
         },
         'params': {
             'skip_download': 'live stream',
@@ -234,29 +217,22 @@ class RTVELiveIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        start_time = time.gmtime()
         video_id = mobj.group('id')
 
         webpage = self._download_webpage(url, video_id)
         title = remove_end(self._og_search_title(webpage), ' en directo en RTVE.es')
         title = remove_start(title, 'Estoy viendo ')
-        title += ' ' + time.strftime('%Y-%m-%dZ%H%M%S', start_time)
 
         vidplayer_id = self._search_regex(
             (r'playerId=player([0-9]+)',
              r'class=["\'].*?\blive_mod\b.*?["\'][^>]+data-assetid=["\'](\d+)',
              r'data-id=["\'](\d+)'),
             webpage, 'internal video ID')
-        png_url = 'http://www.rtve.es/ztnr/movil/thumbnail/amonet/videos/%s.png' % vidplayer_id
-        png = self._download_webpage(png_url, video_id, 'Downloading url information')
-        m3u8_url = _decrypt_url(png)
-        formats = self._extract_m3u8_formats(m3u8_url, video_id, ext='mp4')
-        self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': title,
-            'formats': formats,
+            'title': self._live_title(title),
+            'formats': self._extract_png_formats(vidplayer_id),
             'is_live': True,
         }
 

From 3be098010f667b14075e3dfad1e74e5e2becc8ea Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 14 Mar 2021 20:08:46 +0100
Subject: [PATCH 278/860] [applepodcasts] fix extraction(closes #28445)

---
 youtube_dl/extractor/applepodcasts.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/applepodcasts.py b/youtube_dl/extractor/applepodcasts.py
index 95758fece..6a74de758 100644
--- a/youtube_dl/extractor/applepodcasts.py
+++ b/youtube_dl/extractor/applepodcasts.py
@@ -42,6 +42,7 @@ class ApplePodcastsIE(InfoExtractor):
         ember_data = self._parse_json(self._search_regex(
             r'id="shoebox-ember-data-store"[^>]*>\s*({.+?})\s*<',
             webpage, 'ember data'), episode_id)
+        ember_data = ember_data.get(episode_id) or ember_data
         episode = ember_data['data']['attributes']
         description = episode.get('description') or {}
 

From 357bfe251d7f4f8bb9319bc6531a3813b5a355a8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 15 Mar 2021 20:42:20 +0700
Subject: [PATCH 279/860] [svtplay] Improve extraction (closes #28448)

---
 youtube_dl/extractor/svt.py | 20 +++++++++++++-------
 1 file changed, 13 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index 4acc29fce..aba9bb447 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -146,18 +146,19 @@ class SVTPlayIE(SVTPlayBaseIE):
                         )
                         (?P<svt_id>[^/?#&]+)|
                         https?://(?:www\.)?(?:svtplay|oppetarkiv)\.se/(?:video|klipp|kanaler)/(?P<id>[^/?#&]+)
+                        (?:.*?modalId=(?P<modal_id>[\da-zA-Z-]+))?
                     )
                     '''
     _TESTS = [{
-        'url': 'https://www.svtplay.se/video/26194546/det-har-ar-himlen',
+        'url': 'https://www.svtplay.se/video/30479064',
         'md5': '2382036fd6f8c994856c323fe51c426e',
         'info_dict': {
-            'id': 'jNwpV9P',
+            'id': '8zVbDPA',
             'ext': 'mp4',
-            'title': 'Det här är himlen',
-            'timestamp': 1586044800,
-            'upload_date': '20200405',
-            'duration': 3515,
+            'title': 'Designdrömmar i Stenungsund',
+            'timestamp': 1615770000,
+            'upload_date': '20210315',
+            'duration': 3519,
             'thumbnail': r're:^https?://(?:.*[\.-]jpg|www.svtstatic.se/image/.*)$',
             'age_limit': 0,
             'subtitles': {
@@ -173,6 +174,9 @@ class SVTPlayIE(SVTPlayBaseIE):
             # AssertionError: Expected test_SVTPlay_jNwpV9P.mp4 to be at least 9.77KiB, but it's only 864.00B
             'skip_download': True,
         },
+    }, {
+        'url': 'https://www.svtplay.se/video/30479064/husdrommar/husdrommar-sasong-8-designdrommar-i-stenungsund?modalId=8zVbDPA',
+        'only_matching': True,
     }, {
         # geo restricted to Sweden
         'url': 'http://www.oppetarkiv.se/video/5219710/trollflojten',
@@ -219,7 +223,8 @@ class SVTPlayIE(SVTPlayBaseIE):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        video_id, svt_id = mobj.group('id', 'svt_id')
+        video_id = mobj.group('id')
+        svt_id = mobj.group('svt_id') or mobj.group('modal_id')
 
         if svt_id:
             return self._extract_by_video_id(svt_id)
@@ -254,6 +259,7 @@ class SVTPlayIE(SVTPlayBaseIE):
         if not svt_id:
             svt_id = self._search_regex(
                 (r'<video[^>]+data-video-id=["\']([\da-zA-Z-]+)',
+                 r'<[^>]+\bdata-rt=["\']top-area-play-button["\'][^>]+\bhref=["\'][^"\']*video/%s/[^"\']*\bmodalId=([\da-zA-Z-]+)' % re.escape(video_id),
                  r'["\']videoSvtId["\']\s*:\s*["\']([\da-zA-Z-]+)',
                  r'["\']videoSvtId\\?["\']\s*:\s*\\?["\']([\da-zA-Z-]+)',
                  r'"content"\s*:\s*{.*?"id"\s*:\s*"([\da-zA-Z-]+)"',

From f912d6c8cf5a68d576abe4426e12554c3404a7dd Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 15 Mar 2021 21:43:53 +0100
Subject: [PATCH 280/860] [mlb] fix video extracion(#21241)

---
 youtube_dl/extractor/extractors.py |   5 +-
 youtube_dl/extractor/mlb.py        | 189 +++++++++++++++++++++++++----
 2 files changed, 172 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 07a8af055..c2f67323b 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -673,7 +673,10 @@ from .mixcloud import (
     MixcloudUserIE,
     MixcloudPlaylistIE,
 )
-from .mlb import MLBIE
+from .mlb import (
+    MLBIE,
+    MLBVideoIE,
+)
 from .mnet import MnetIE
 from .moevideo import MoeVideoIE
 from .mofosex import (
diff --git a/youtube_dl/extractor/mlb.py b/youtube_dl/extractor/mlb.py
index b907f6b49..b69301d97 100644
--- a/youtube_dl/extractor/mlb.py
+++ b/youtube_dl/extractor/mlb.py
@@ -1,15 +1,91 @@
 from __future__ import unicode_literals
 
-from .nhl import NHLBaseIE
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    parse_duration,
+    parse_iso8601,
+    try_get,
+)
 
 
-class MLBIE(NHLBaseIE):
+class MLBBaseIE(InfoExtractor):
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        video = self._download_video_data(display_id)
+        video_id = video['id']
+        title = video['title']
+        feed = self._get_feed(video)
+
+        formats = []
+        for playback in (feed.get('playbacks') or []):
+            playback_url = playback.get('url')
+            if not playback_url:
+                continue
+            name = playback.get('name')
+            ext = determine_ext(playback_url)
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    playback_url, video_id, 'mp4',
+                    'm3u8_native', m3u8_id=name, fatal=False))
+            else:
+                f = {
+                    'format_id': name,
+                    'url': playback_url,
+                }
+                mobj = re.search(r'_(\d+)K_(\d+)X(\d+)', name)
+                if mobj:
+                    f.update({
+                        'height': int(mobj.group(3)),
+                        'tbr': int(mobj.group(1)),
+                        'width': int(mobj.group(2)),
+                    })
+                mobj = re.search(r'_(\d+)x(\d+)_(\d+)_(\d+)K\.mp4', playback_url)
+                if mobj:
+                    f.update({
+                        'fps': int(mobj.group(3)),
+                        'height': int(mobj.group(2)),
+                        'tbr': int(mobj.group(4)),
+                        'width': int(mobj.group(1)),
+                    })
+                formats.append(f)
+        self._sort_formats(formats)
+
+        thumbnails = []
+        for cut in (try_get(feed, lambda x: x['image']['cuts'], list) or []):
+            src = cut.get('src')
+            if not src:
+                continue
+            thumbnails.append({
+                'height': int_or_none(cut.get('height')),
+                'url': src,
+                'width': int_or_none(cut.get('width')),
+            })
+
+        language = (video.get('language') or 'EN').lower()
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'description': video.get('description'),
+            'duration': parse_duration(feed.get('duration')),
+            'thumbnails': thumbnails,
+            'timestamp': parse_iso8601(video.get(self._TIMESTAMP_KEY)),
+            'subtitles': self._extract_mlb_subtitles(feed, language),
+        }
+
+
+class MLBIE(MLBBaseIE):
     _VALID_URL = r'''(?x)
                     https?://
-                        (?:[\da-z_-]+\.)*(?P<site>mlb)\.com/
+                        (?:[\da-z_-]+\.)*mlb\.com/
                         (?:
                             (?:
-                                (?:[^/]+/)*c-|
+                                (?:[^/]+/)*video/[^/]+/c-|
                                 (?:
                                     shared/video/embed/(?:embed|m-internal-embed)\.html|
                                     (?:[^/]+/)+(?:play|index)\.jsp|
@@ -18,7 +94,6 @@ class MLBIE(NHLBaseIE):
                             (?P<id>\d+)
                         )
                     '''
-    _CONTENT_DOMAIN = 'content.mlb.com'
     _TESTS = [
         {
             'url': 'https://www.mlb.com/mariners/video/ackleys-spectacular-catch/c-34698933',
@@ -76,18 +151,6 @@ class MLBIE(NHLBaseIE):
                 'thumbnail': r're:^https?://.*\.jpg$',
             },
         },
-        {
-            'url': 'https://www.mlb.com/news/blue-jays-kevin-pillar-goes-spidey-up-the-wall-to-rob-tim-beckham-of-a-homer/c-118550098',
-            'md5': 'e09e37b552351fddbf4d9e699c924d68',
-            'info_dict': {
-                'id': '75609783',
-                'ext': 'mp4',
-                'title': 'Must C: Pillar climbs for catch',
-                'description': '4/15/15: Blue Jays outfielder Kevin Pillar continues his defensive dominance by climbing the wall in left to rob Tim Beckham of a home run',
-                'timestamp': 1429139220,
-                'upload_date': '20150415',
-            }
-        },
         {
             'url': 'https://www.mlb.com/video/hargrove-homers-off-caldwell/c-1352023483?tid=67793694',
             'only_matching': True,
@@ -113,8 +176,92 @@ class MLBIE(NHLBaseIE):
             'url': 'http://mlb.mlb.com/shared/video/embed/m-internal-embed.html?content_id=75609783&property=mlb&autoplay=true&hashmode=false&siteSection=mlb/multimedia/article_118550098/article_embed&club=mlb',
             'only_matching': True,
         },
-        {
-            'url': 'https://www.mlb.com/cut4/carlos-gomez-borrowed-sunglasses-from-an-as-fan/c-278912842',
-            'only_matching': True,
-        }
     ]
+    _TIMESTAMP_KEY = 'date'
+
+    @staticmethod
+    def _get_feed(video):
+        return video
+
+    @staticmethod
+    def _extract_mlb_subtitles(feed, language):
+        subtitles = {}
+        for keyword in (feed.get('keywordsAll') or []):
+            keyword_type = keyword.get('type')
+            if keyword_type and keyword_type.startswith('closed_captions_location_'):
+                cc_location = keyword.get('value')
+                if cc_location:
+                    subtitles.setdefault(language, []).append({
+                        'url': cc_location,
+                    })
+        return subtitles
+
+    def _download_video_data(self, display_id):
+        return self._download_json(
+            'http://content.mlb.com/mlb/item/id/v1/%s/details/web-v1.json' % display_id,
+            display_id)
+
+
+class MLBVideoIE(MLBBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?mlb\.com/(?:[^/]+/)*video/(?P<id>[^/?&#]+)'
+    _TEST = {
+        'url': 'https://www.mlb.com/mariners/video/ackley-s-spectacular-catch-c34698933',
+        'md5': '632358dacfceec06bad823b83d21df2d',
+        'info_dict': {
+            'id': 'c04a8863-f569-42e6-9f87-992393657614',
+            'ext': 'mp4',
+            'title': "Ackley's spectacular catch",
+            'description': 'md5:7f5a981eb4f3cbc8daf2aeffa2215bf0',
+            'duration': 66,
+            'timestamp': 1405995000,
+            'upload_date': '20140722',
+            'thumbnail': r're:^https?://.+',
+        },
+    }
+    _TIMESTAMP_KEY = 'timestamp'
+
+    @classmethod
+    def suitable(cls, url):
+        return False if MLBIE.suitable(url) else super(MLBVideoIE, cls).suitable(url)
+
+    @staticmethod
+    def _get_feed(video):
+        return video['feeds'][0]
+
+    @staticmethod
+    def _extract_mlb_subtitles(feed, language):
+        subtitles = {}
+        for cc_location in (feed.get('closedCaptions') or []):
+            subtitles.setdefault(language, []).append({
+                'url': cc_location,
+            })
+
+    def _download_video_data(self, display_id):
+        # https://www.mlb.com/data-service/en/videos/[SLUG]
+        return self._download_json(
+            'https://fastball-gateway.mlb.com/graphql',
+            display_id, query={
+                'query': '''{
+  mediaPlayback(ids: "%s") {
+    description
+    feeds(types: CMS) {
+      closedCaptions
+      duration
+      image {
+        cuts {
+          width
+          height
+          src
+        }
+      }
+      playbacks {
+        name
+        url
+      }
+    }
+    id
+    timestamp
+    title
+  }
+}''' % display_id,
+            })['data']['mediaPlayback'][0]

From fa6bf0a7112e83d36567072985d56440bb34de72 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 19 Mar 2021 12:37:22 +0100
Subject: [PATCH 281/860] [vvvvid] fix kenc format extraction(closes #28473)

---
 youtube_dl/extractor/vvvvid.py | 25 ++++++++++++++-----------
 1 file changed, 14 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/vvvvid.py b/youtube_dl/extractor/vvvvid.py
index 7c94c4ee2..bc196f8a0 100644
--- a/youtube_dl/extractor/vvvvid.py
+++ b/youtube_dl/extractor/vvvvid.py
@@ -182,17 +182,20 @@ class VVVVIDIE(InfoExtractor):
             if not embed_code:
                 continue
             embed_code = ds(embed_code)
-            if video_type in ('video/rcs', 'video/kenc'):
-                if video_type == 'video/kenc':
-                    kenc = self._download_json(
-                        'https://www.vvvvid.it/kenc', video_id, query={
-                            'action': 'kt',
-                            'conn_id': self._conn_id,
-                            'url': embed_code,
-                        }, fatal=False) or {}
-                    kenc_message = kenc.get('message')
-                    if kenc_message:
-                        embed_code += '?' + ds(kenc_message)
+            if video_type == 'video/kenc':
+                embed_code = re.sub(r'https?(://[^/]+)/z/', r'https\1/i/', embed_code).replace('/manifest.f4m', '/master.m3u8')
+                kenc = self._download_json(
+                    'https://www.vvvvid.it/kenc', video_id, query={
+                        'action': 'kt',
+                        'conn_id': self._conn_id,
+                        'url': embed_code,
+                    }, fatal=False) or {}
+                kenc_message = kenc.get('message')
+                if kenc_message:
+                    embed_code += '?' + ds(kenc_message)
+                formats.extend(self._extract_m3u8_formats(
+                    embed_code, video_id, 'mp4', m3u8_id='hls', fatal=False))
+            elif video_type == 'video/rcs':
                 formats.extend(self._extract_akamai_formats(embed_code, video_id))
             elif video_type == 'video/youtube':
                 info.update({

From 7e79ba7dd6e6649dd2ce3a74004b2044f2182881 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 20 Mar 2021 05:45:36 +0700
Subject: [PATCH 282/860] [vimeo:album] Fix extraction for albums with number
 of videos multiple to page size (closes #28486)

---
 youtube_dl/extractor/vimeo.py | 14 +++++++++-----
 1 file changed, 9 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index bd2663fe0..955651bec 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -939,11 +939,15 @@ class VimeoAlbumIE(VimeoBaseInfoExtractor):
         }
         if hashed_pass:
             query['_hashed_pass'] = hashed_pass
-        videos = self._download_json(
-            'https://api.vimeo.com/albums/%s/videos' % album_id,
-            album_id, 'Downloading page %d' % api_page, query=query, headers={
-                'Authorization': 'jwt ' + authorization,
-            })['data']
+        try:
+            videos = self._download_json(
+                'https://api.vimeo.com/albums/%s/videos' % album_id,
+                album_id, 'Downloading page %d' % api_page, query=query, headers={
+                    'Authorization': 'jwt ' + authorization,
+                })['data']
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 400:
+                return
         for video in videos:
             link = video.get('link')
             if not link:

From 21ccd0d7f46002acc61eb21bd0d4e492064c7fe1 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 21 Mar 2021 09:10:38 +0100
Subject: [PATCH 283/860] [tiktok] detect private videos(closes #28453)

---
 youtube_dl/extractor/tiktok.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/tiktok.py b/youtube_dl/extractor/tiktok.py
index ea1beb8af..4faa6de54 100644
--- a/youtube_dl/extractor/tiktok.py
+++ b/youtube_dl/extractor/tiktok.py
@@ -107,9 +107,12 @@ class TikTokIE(TikTokBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
-        data = self._parse_json(self._search_regex(
+        page_props = self._parse_json(self._search_regex(
             r'<script[^>]+\bid=["\']__NEXT_DATA__[^>]+>\s*({.+?})\s*</script',
-            webpage, 'data'), video_id)['props']['pageProps']['itemInfo']['itemStruct']
+            webpage, 'data'), video_id)['props']['pageProps']
+        data = try_get(page_props, lambda x: x['itemInfo']['itemStruct'], dict)
+        if not data and page_props.get('statusCode') == 10216:
+            raise ExtractorError('This video is private', expected=True)
         return self._extract_video(data, video_id)
 
 

From 00b4d72d1e5c6e8151f33c822f1f9165ea99dfc8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Martin=20Str=C3=B6m?= <name@my-domain.se>
Date: Mon, 22 Mar 2021 14:56:58 +0100
Subject: [PATCH 284/860] [vgtv] Add support for new tv.aftonbladet.se URL
 schema (#28514)

Co-authored-by: Sergey M <dstftw@gmail.com>
---
 youtube_dl/extractor/vgtv.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/vgtv.py b/youtube_dl/extractor/vgtv.py
index fe7a26b62..22e99e8f0 100644
--- a/youtube_dl/extractor/vgtv.py
+++ b/youtube_dl/extractor/vgtv.py
@@ -23,6 +23,8 @@ class VGTVIE(XstreamIE):
         'fvn.no/fvntv': 'fvntv',
         'aftenposten.no/webtv': 'aptv',
         'ap.vgtv.no/webtv': 'aptv',
+        'tv.aftonbladet.se': 'abtv',
+        # obsolete URL schemas, kept in order to save one HTTP redirect
         'tv.aftonbladet.se/abtv': 'abtv',
         'www.aftonbladet.se/tv': 'abtv',
     }
@@ -140,6 +142,10 @@ class VGTVIE(XstreamIE):
             'url': 'http://www.vgtv.no/#!/video/127205/inside-the-mind-of-favela-funk',
             'only_matching': True,
         },
+        {
+            'url': 'https://tv.aftonbladet.se/video/36015/vulkanutbrott-i-rymden-nu-slapper-nasa-bilderna',
+            'only_matching': True,
+        },
         {
             'url': 'http://tv.aftonbladet.se/abtv/articles/36015',
             'only_matching': True,

From 8117d613acdd0a2874e52bfa52c3574f46e3a4fb Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 22 Mar 2021 15:58:44 +0100
Subject: [PATCH 285/860] [zingmp3] fix extraction(closes #11589, closes
 #16409, closes #16968, closes #27205)

---
 youtube_dl/extractor/extractors.py |   5 +-
 youtube_dl/extractor/zingmp3.py    | 212 ++++++++++++++++-------------
 2 files changed, 119 insertions(+), 98 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index c2f67323b..8b55947f6 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1624,5 +1624,8 @@ from .zattoo import (
 )
 from .zdf import ZDFIE, ZDFChannelIE
 from .zhihu import ZhihuIE
-from .zingmp3 import ZingMp3IE
+from .zingmp3 import (
+    ZingMp3IE,
+    ZingMp3AlbumIE,
+)
 from .zype import ZypeIE
diff --git a/youtube_dl/extractor/zingmp3.py b/youtube_dl/extractor/zingmp3.py
index adfdcaabf..207c04f5e 100644
--- a/youtube_dl/extractor/zingmp3.py
+++ b/youtube_dl/extractor/zingmp3.py
@@ -1,93 +1,94 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
     int_or_none,
-    update_url_query,
 )
 
 
-class ZingMp3BaseInfoExtractor(InfoExtractor):
+class ZingMp3BaseIE(InfoExtractor):
+    _VALID_URL_TMPL = r'https?://(?:mp3\.zing|zingmp3)\.vn/(?:%s)/[^/]+/(?P<id>\w+)\.html'
+    _GEO_COUNTRIES = ['VN']
 
-    def _extract_item(self, item, page_type, fatal=True):
-        error_message = item.get('msg')
-        if error_message:
-            if not fatal:
-                return
-            raise ExtractorError(
-                '%s returned error: %s' % (self.IE_NAME, error_message),
-                expected=True)
+    def _extract_item(self, item, fatal):
+        item_id = item['id']
+        title = item.get('name') or item['title']
 
         formats = []
-        for quality, source_url in zip(item.get('qualities') or item.get('quality', []), item.get('source_list') or item.get('source', [])):
-            if not source_url or source_url == 'require vip':
+        for k, v in (item.get('source') or {}).items():
+            if not v:
                 continue
-            if not re.match(r'https?://', source_url):
-                source_url = '//' + source_url
-            source_url = self._proto_relative_url(source_url, 'http:')
-            quality_num = int_or_none(quality)
-            f = {
-                'format_id': quality,
-                'url': source_url,
-            }
-            if page_type == 'video':
-                f.update({
-                    'height': quality_num,
-                    'ext': 'mp4',
-                })
+            if k in ('mp4', 'hls'):
+                for res, video_url in v.items():
+                    if not video_url:
+                        continue
+                    if k == 'hls':
+                        formats.extend(self._extract_m3u8_formats(
+                            video_url, item_id, 'mp4',
+                            'm3u8_native', m3u8_id=k, fatal=False))
+                    elif k == 'mp4':
+                        formats.append({
+                            'format_id': 'mp4-' + res,
+                            'url': video_url,
+                            'height': int_or_none(self._search_regex(
+                                r'^(\d+)p', res, 'resolution', default=None)),
+                        })
             else:
-                f.update({
-                    'abr': quality_num,
+                formats.append({
                     'ext': 'mp3',
+                    'format_id': k,
+                    'tbr': int_or_none(k),
+                    'url': self._proto_relative_url(v),
+                    'vcodec': 'none',
                 })
-            formats.append(f)
+        if not formats:
+            if not fatal:
+                return
+            msg = item['msg']
+            if msg == 'Sorry, this content is not available in your country.':
+                self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
+            raise ExtractorError(msg, expected=True)
+        self._sort_formats(formats)
 
-        cover = item.get('cover')
+        subtitles = None
+        lyric = item.get('lyric')
+        if lyric:
+            subtitles = {
+                'origin': [{
+                    'url': lyric,
+                }],
+            }
+
+        album = item.get('album') or {}
 
         return {
-            'title': (item.get('name') or item.get('title')).strip(),
+            'id': item_id,
+            'title': title,
             'formats': formats,
-            'thumbnail': 'http:/' + cover if cover else None,
-            'artist': item.get('artist'),
+            'thumbnail': item.get('thumbnail'),
+            'subtitles': subtitles,
+            'duration': int_or_none(item.get('duration')),
+            'track': title,
+            'artist': item.get('artists_names'),
+            'album': album.get('name') or album.get('title'),
+            'album_artist': album.get('artists_names'),
         }
 
-    def _extract_player_json(self, player_json_url, id, page_type, playlist_title=None):
-        player_json = self._download_json(player_json_url, id, 'Downloading Player JSON')
-        items = player_json['data']
-        if 'item' in items:
-            items = items['item']
-
-        if len(items) == 1:
-            # one single song
-            data = self._extract_item(items[0], page_type)
-            data['id'] = id
-
-            return data
-        else:
-            # playlist of songs
-            entries = []
-
-            for i, item in enumerate(items, 1):
-                entry = self._extract_item(item, page_type, fatal=False)
-                if not entry:
-                    continue
-                entry['id'] = '%s-%d' % (id, i)
-                entries.append(entry)
-
-            return {
-                '_type': 'playlist',
-                'id': id,
-                'title': playlist_title,
-                'entries': entries,
-            }
+    def _real_extract(self, url):
+        page_id = self._match_id(url)
+        webpage = self._download_webpage(
+            url.replace('://zingmp3.vn/', '://mp3.zing.vn/'),
+            page_id, query={'play_song': 1})
+        data_path = self._search_regex(
+            r'data-xml="([^"]+)', webpage, 'data path')
+        return self._process_data(self._download_json(
+            'https://mp3.zing.vn/xhr' + data_path, page_id)['data'])
 
 
-class ZingMp3IE(ZingMp3BaseInfoExtractor):
-    _VALID_URL = r'https?://mp3\.zing\.vn/(?:bai-hat|album|playlist|video-clip)/[^/]+/(?P<id>\w+)\.html'
+class ZingMp3IE(ZingMp3BaseIE):
+    _VALID_URL = ZingMp3BaseIE._VALID_URL_TMPL % 'bai-hat|video-clip'
     _TESTS = [{
         'url': 'http://mp3.zing.vn/bai-hat/Xa-Mai-Xa-Bao-Thy/ZWZB9WAB.html',
         'md5': 'ead7ae13693b3205cbc89536a077daed',
@@ -95,49 +96,66 @@ class ZingMp3IE(ZingMp3BaseInfoExtractor):
             'id': 'ZWZB9WAB',
             'title': 'Xa Mãi Xa',
             'ext': 'mp3',
-            'thumbnail': r're:^https?://.*\.jpg$',
+            'thumbnail': r're:^https?://.+\.jpg',
+            'subtitles': {
+                'origin': [{
+                    'ext': 'lrc',
+                }]
+            },
+            'duration': 255,
+            'track': 'Xa Mãi Xa',
+            'artist': 'Bảo Thy',
+            'album': 'Special Album',
+            'album_artist': 'Bảo Thy',
         },
     }, {
-        'url': 'http://mp3.zing.vn/video-clip/Let-It-Go-Frozen-OST-Sungha-Jung/ZW6BAEA0.html',
-        'md5': '870295a9cd8045c0e15663565902618d',
+        'url': 'https://mp3.zing.vn/video-clip/Suong-Hoa-Dua-Loi-K-ICM-RYO/ZO8ZF7C7.html',
+        'md5': 'e9c972b693aa88301ef981c8151c4343',
         'info_dict': {
-            'id': 'ZW6BAEA0',
-            'title': 'Let It Go (Frozen OST)',
+            'id': 'ZO8ZF7C7',
+            'title': 'Sương Hoa Đưa Lối',
             'ext': 'mp4',
+            'thumbnail': r're:^https?://.+\.jpg',
+            'duration': 207,
+            'track': 'Sương Hoa Đưa Lối',
+            'artist': 'K-ICM, RYO',
         },
     }, {
-        'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
-        'info_dict': {
-            '_type': 'playlist',
-            'id': 'ZWZBWDAF',
-            'title': 'Lâu Đài Tình Ái - Bằng Kiều,Minh Tuyết | Album 320 lossless',
-        },
-        'playlist_count': 10,
-        'skip': 'removed at the request of the owner',
-    }, {
-        'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
+        'url': 'https://zingmp3.vn/bai-hat/Xa-Mai-Xa-Bao-Thy/ZWZB9WAB.html',
         'only_matching': True,
     }]
     IE_NAME = 'zingmp3'
     IE_DESC = 'mp3.zing.vn'
 
-    def _real_extract(self, url):
-        page_id = self._match_id(url)
+    def _process_data(self, data):
+        return self._extract_item(data, True)
 
-        webpage = self._download_webpage(url, page_id)
 
-        player_json_url = self._search_regex([
-            r'data-xml="([^"]+)',
-            r'&amp;xmlURL=([^&]+)&'
-        ], webpage, 'player xml url')
+class ZingMp3AlbumIE(ZingMp3BaseIE):
+    _VALID_URL = ZingMp3BaseIE._VALID_URL_TMPL % 'album|playlist'
+    _TESTS = [{
+        'url': 'http://mp3.zing.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
+        'info_dict': {
+            '_type': 'playlist',
+            'id': 'ZWZBWDAF',
+            'title': 'Lâu Đài Tình Ái',
+        },
+        'playlist_count': 10,
+    }, {
+        'url': 'http://mp3.zing.vn/playlist/Duong-Hong-Loan-apollobee/IWCAACCB.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://zingmp3.vn/album/Lau-Dai-Tinh-Ai-Bang-Kieu-Minh-Tuyet/ZWZBWDAF.html',
+        'only_matching': True,
+    }]
+    IE_NAME = 'zingmp3:album'
 
-        playlist_title = None
-        page_type = self._search_regex(r'/(?:html5)?xml/([^/-]+)', player_json_url, 'page type')
-        if page_type == 'video':
-            player_json_url = update_url_query(player_json_url, {'format': 'json'})
-        else:
-            player_json_url = player_json_url.replace('/xml/', '/html5xml/')
-            if page_type == 'album':
-                playlist_title = self._og_search_title(webpage)
-
-        return self._extract_player_json(player_json_url, page_id, page_type, playlist_title)
+    def _process_data(self, data):
+        def entries():
+            for item in (data.get('items') or []):
+                entry = self._extract_item(item, False)
+                if entry:
+                    yield entry
+        info = data.get('info') or {}
+        return self.playlist_result(
+            entries(), info.get('id'), info.get('name') or info.get('title'))

From 5208ae92fc3e2916cdccae45c6b9a516be3d5796 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 24 Mar 2021 02:57:35 +0700
Subject: [PATCH 286/860] [youtube] Fix default value for
 youtube_include_dash_manifest (closes #28523)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index eb5a58807..badca3977 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1617,7 +1617,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     f['format_id'] = itag
                 formats.append(f)
 
-        if self._downloader.params.get('youtube_include_dash_manifest'):
+        if self._downloader.params.get('youtube_include_dash_manifest', True):
             dash_manifest_url = streaming_data.get('dashManifestUrl')
             if dash_manifest_url:
                 for f in self._extract_mpd_formats(

From a40002444e64957594a1305bb2740fddb477beeb Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 24 Mar 2021 15:10:19 +0100
Subject: [PATCH 287/860] [bbc] fix BBC IPlayer Episodes/Group
 extraction(closes #28360)

---
 youtube_dl/extractor/bbc.py        | 205 ++++++++++++++++++++++++++---
 youtube_dl/extractor/extractors.py |   3 +-
 2 files changed, 192 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 92e6f1bea..e8d000bbb 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -1,17 +1,22 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import functools
 import itertools
+import json
 import re
 
 from .common import InfoExtractor
 from ..compat import (
     compat_etree_Element,
     compat_HTTPError,
+    compat_parse_qs,
+    compat_urllib_parse_urlparse,
     compat_urlparse,
 )
 from ..utils import (
     ExtractorError,
+    OnDemandPagedList,
     clean_html,
     dict_get,
     float_or_none,
@@ -811,7 +816,7 @@ class BBCIE(BBCCoUkIE):
 
     @classmethod
     def suitable(cls, url):
-        EXCLUDE_IE = (BBCCoUkIE, BBCCoUkArticleIE, BBCCoUkIPlayerPlaylistIE, BBCCoUkPlaylistIE)
+        EXCLUDE_IE = (BBCCoUkIE, BBCCoUkArticleIE, BBCCoUkIPlayerEpisodesIE, BBCCoUkIPlayerGroupIE, BBCCoUkPlaylistIE)
         return (False if any(ie.suitable(url) for ie in EXCLUDE_IE)
                 else super(BBCIE, cls).suitable(url))
 
@@ -1338,21 +1343,149 @@ class BBCCoUkPlaylistBaseIE(InfoExtractor):
             playlist_id, title, description)
 
 
-class BBCCoUkIPlayerPlaylistIE(BBCCoUkPlaylistBaseIE):
-    IE_NAME = 'bbc.co.uk:iplayer:playlist'
-    _VALID_URL = r'https?://(?:www\.)?bbc\.co\.uk/iplayer/(?:episodes|group)/(?P<id>%s)' % BBCCoUkIE._ID_REGEX
-    _URL_TEMPLATE = 'http://www.bbc.co.uk/iplayer/episode/%s'
-    _VIDEO_ID_TEMPLATE = r'data-ip-id=["\'](%s)'
+class BBCCoUkIPlayerPlaylistBaseIE(InfoExtractor):
+    _VALID_URL_TMPL = r'https?://(?:www\.)?bbc\.co\.uk/iplayer/%%s/(?P<id>%s)' % BBCCoUkIE._ID_REGEX
+
+    @staticmethod
+    def _get_default(episode, key, default_key='default'):
+        return try_get(episode, lambda x: x[key][default_key])
+
+    def _get_description(self, data):
+        synopsis = data.get(self._DESCRIPTION_KEY) or {}
+        return dict_get(synopsis, ('large', 'medium', 'small'))
+
+    def _fetch_page(self, programme_id, per_page, series_id, page):
+        elements = self._get_elements(self._call_api(
+            programme_id, per_page, page + 1, series_id))
+        for element in elements:
+            episode = self._get_episode(element)
+            episode_id = episode.get('id')
+            if not episode_id:
+                continue
+            thumbnail = None
+            image = self._get_episode_image(episode)
+            if image:
+                thumbnail = image.replace('{recipe}', 'raw')
+            category = self._get_default(episode, 'labels', 'category')
+            yield {
+                '_type': 'url',
+                'id': episode_id,
+                'title': self._get_episode_field(episode, 'subtitle'),
+                'url': 'https://www.bbc.co.uk/iplayer/episode/' + episode_id,
+                'thumbnail': thumbnail,
+                'description': self._get_description(episode),
+                'categories': [category] if category else None,
+                'series': self._get_episode_field(episode, 'title'),
+                'ie_key': BBCCoUkIE.ie_key(),
+            }
+
+    def _real_extract(self, url):
+        pid = self._match_id(url)
+        qs = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+        series_id = qs.get('seriesId', [None])[0]
+        page = qs.get('page', [None])[0]
+        per_page = 36 if page else self._PAGE_SIZE
+        fetch_page = functools.partial(self._fetch_page, pid, per_page, series_id)
+        entries = fetch_page(int(page) - 1) if page else OnDemandPagedList(fetch_page, self._PAGE_SIZE)
+        playlist_data = self._get_playlist_data(self._call_api(pid, 1))
+        return self.playlist_result(
+            entries, pid, self._get_playlist_title(playlist_data),
+            self._get_description(playlist_data))
+
+
+class BBCCoUkIPlayerEpisodesIE(BBCCoUkIPlayerPlaylistBaseIE):
+    IE_NAME = 'bbc.co.uk:iplayer:episodes'
+    _VALID_URL = BBCCoUkIPlayerPlaylistBaseIE._VALID_URL_TMPL % 'episodes'
     _TESTS = [{
         'url': 'http://www.bbc.co.uk/iplayer/episodes/b05rcz9v',
         'info_dict': {
             'id': 'b05rcz9v',
             'title': 'The Disappearance',
-            'description': 'French thriller serial about a missing teenager.',
+            'description': 'md5:58eb101aee3116bad4da05f91179c0cb',
         },
-        'playlist_mincount': 6,
-        'skip': 'This programme is not currently available on BBC iPlayer',
+        'playlist_mincount': 8,
     }, {
+        # all seasons
+        'url': 'https://www.bbc.co.uk/iplayer/episodes/b094m5t9/doctor-foster',
+        'info_dict': {
+            'id': 'b094m5t9',
+            'title': 'Doctor Foster',
+            'description': 'md5:5aa9195fad900e8e14b52acd765a9fd6',
+        },
+        'playlist_mincount': 10,
+    }, {
+        # explicit season
+        'url': 'https://www.bbc.co.uk/iplayer/episodes/b094m5t9/doctor-foster?seriesId=b094m6nv',
+        'info_dict': {
+            'id': 'b094m5t9',
+            'title': 'Doctor Foster',
+            'description': 'md5:5aa9195fad900e8e14b52acd765a9fd6',
+        },
+        'playlist_mincount': 5,
+    }, {
+        # all pages
+        'url': 'https://www.bbc.co.uk/iplayer/episodes/m0004c4v/beechgrove',
+        'info_dict': {
+            'id': 'm0004c4v',
+            'title': 'Beechgrove',
+            'description': 'Gardening show that celebrates Scottish horticulture and growing conditions.',
+        },
+        'playlist_mincount': 37,
+    }, {
+        # explicit page
+        'url': 'https://www.bbc.co.uk/iplayer/episodes/m0004c4v/beechgrove?page=2',
+        'info_dict': {
+            'id': 'm0004c4v',
+            'title': 'Beechgrove',
+            'description': 'Gardening show that celebrates Scottish horticulture and growing conditions.',
+        },
+        'playlist_mincount': 1,
+    }]
+    _PAGE_SIZE = 100
+    _DESCRIPTION_KEY = 'synopsis'
+
+    def _get_episode_image(self, episode):
+        return self._get_default(episode, 'image')
+
+    def _get_episode_field(self, episode, field):
+        return self._get_default(episode, field)
+
+    @staticmethod
+    def _get_elements(data):
+        return data['entities']['results']
+
+    @staticmethod
+    def _get_episode(element):
+        return element.get('episode') or {}
+
+    def _call_api(self, pid, per_page, page=1, series_id=None):
+        variables = {
+            'id': pid,
+            'page': page,
+            'perPage': per_page,
+        }
+        if series_id:
+            variables['sliceId'] = series_id
+        return self._download_json(
+            'https://graph.ibl.api.bbc.co.uk/', pid, headers={
+                'Content-Type': 'application/json'
+            }, data=json.dumps({
+                'id': '5692d93d5aac8d796a0305e895e61551',
+                'variables': variables,
+            }).encode('utf-8'))['data']['programme']
+
+    @staticmethod
+    def _get_playlist_data(data):
+        return data
+
+    def _get_playlist_title(self, data):
+        return self._get_default(data, 'title')
+
+
+class BBCCoUkIPlayerGroupIE(BBCCoUkIPlayerPlaylistBaseIE):
+    IE_NAME = 'bbc.co.uk:iplayer:group'
+    _VALID_URL = BBCCoUkIPlayerPlaylistBaseIE._VALID_URL_TMPL % 'group'
+    _TESTS = [{
         # Available for over a year unlike 30 days for most other programmes
         'url': 'http://www.bbc.co.uk/iplayer/group/p02tcc32',
         'info_dict': {
@@ -1361,14 +1494,56 @@ class BBCCoUkIPlayerPlaylistIE(BBCCoUkPlaylistBaseIE):
             'description': 'md5:683e901041b2fe9ba596f2ab04c4dbe7',
         },
         'playlist_mincount': 10,
+    }, {
+        # all pages
+        'url': 'https://www.bbc.co.uk/iplayer/group/p081d7j7',
+        'info_dict': {
+            'id': 'p081d7j7',
+            'title': 'Music in Scotland',
+            'description': 'Perfomances in Scotland and programmes featuring Scottish acts.',
+        },
+        'playlist_mincount': 47,
+    }, {
+        # explicit page
+        'url': 'https://www.bbc.co.uk/iplayer/group/p081d7j7?page=2',
+        'info_dict': {
+            'id': 'p081d7j7',
+            'title': 'Music in Scotland',
+            'description': 'Perfomances in Scotland and programmes featuring Scottish acts.',
+        },
+        'playlist_mincount': 11,
     }]
+    _PAGE_SIZE = 200
+    _DESCRIPTION_KEY = 'synopses'
 
-    def _extract_title_and_description(self, webpage):
-        title = self._search_regex(r'<h1>([^<]+)</h1>', webpage, 'title', fatal=False)
-        description = self._search_regex(
-            r'<p[^>]+class=(["\'])subtitle\1[^>]*>(?P<value>[^<]+)</p>',
-            webpage, 'description', fatal=False, group='value')
-        return title, description
+    def _get_episode_image(self, episode):
+        return self._get_default(episode, 'images', 'standard')
+
+    def _get_episode_field(self, episode, field):
+        return episode.get(field)
+
+    @staticmethod
+    def _get_elements(data):
+        return data['elements']
+
+    @staticmethod
+    def _get_episode(element):
+        return element
+
+    def _call_api(self, pid, per_page, page=1, series_id=None):
+        return self._download_json(
+            'http://ibl.api.bbc.co.uk/ibl/v1/groups/%s/episodes' % pid,
+            pid, query={
+                'page': page,
+                'per_page': per_page,
+            })['group_episodes']
+
+    @staticmethod
+    def _get_playlist_data(data):
+        return data['group']
+
+    def _get_playlist_title(self, data):
+        return data.get('title')
 
 
 class BBCCoUkPlaylistIE(BBCCoUkPlaylistBaseIE):
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 8b55947f6..e0fd0b648 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -95,7 +95,8 @@ from .bandcamp import BandcampIE, BandcampAlbumIE, BandcampWeeklyIE
 from .bbc import (
     BBCCoUkIE,
     BBCCoUkArticleIE,
-    BBCCoUkIPlayerPlaylistIE,
+    BBCCoUkIPlayerEpisodesIE,
+    BBCCoUkIPlayerGroupIE,
     BBCCoUkPlaylistIE,
     BBCIE,
 )

From eafcadea261dba64c44c5c17ea8a47ac17256617 Mon Sep 17 00:00:00 2001
From: The Hatsune Daishi <nao20010128@gmail.com>
Date: Wed, 24 Mar 2021 23:33:19 +0900
Subject: [PATCH 288/860] [extractor] escape forgotten dot for hostnames in
 regular expression (#28530)

---
 youtube_dl/extractor/generic.py | 2 +-
 youtube_dl/extractor/mtv.py     | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index c2b1b3bdf..f99d887ca 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2953,7 +2953,7 @@ class GenericIE(InfoExtractor):
             webpage)
         if not mobj:
             mobj = re.search(
-                r'data-video-link=["\'](?P<url>http://m.mlb.com/video/[^"\']+)',
+                r'data-video-link=["\'](?P<url>http://m\.mlb\.com/video/[^"\']+)',
                 webpage)
         if mobj is not None:
             return self.url_result(mobj.group('url'), 'MLB')
diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index f5e30d22d..600cf2d89 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -320,7 +320,7 @@ class MTVServicesEmbeddedIE(MTVServicesInfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//media.mtvnservices.com/embed/.+?)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>(?:https?:)?//media\.mtvnservices\.com/embed/.+?)\1', webpage)
         if mobj:
             return mobj.group('url')
 

From d1069d33b4ad3987acc2452756459065ce635d68 Mon Sep 17 00:00:00 2001
From: Roman Sebastian Karwacik <roman.karwacik@rwth-aachen.de>
Date: Sun, 12 Apr 2020 23:27:58 +0200
Subject: [PATCH 289/860] [zoom] Add new extractor(closes #16597, closes
 #27002, closes #28531)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/zoom.py       | 68 ++++++++++++++++++++++++++++++
 2 files changed, 69 insertions(+)
 create mode 100644 youtube_dl/extractor/zoom.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e0fd0b648..b2b39e4dd 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1629,4 +1629,5 @@ from .zingmp3 import (
     ZingMp3IE,
     ZingMp3AlbumIE,
 )
+from .zoom import ZoomIE
 from .zype import ZypeIE
diff --git a/youtube_dl/extractor/zoom.py b/youtube_dl/extractor/zoom.py
new file mode 100644
index 000000000..db073d91d
--- /dev/null
+++ b/youtube_dl/extractor/zoom.py
@@ -0,0 +1,68 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    js_to_json,
+    parse_filesize,
+    urlencode_postdata,
+)
+
+
+class ZoomIE(InfoExtractor):
+    IE_NAME = 'zoom'
+    _VALID_URL = r'(?P<base_url>https?://(?:[^.]+\.)?zoom.us/)rec(?:ording)?/(?:play|share)/(?P<id>[A-Za-z0-9_.-]+)'
+    _TEST = {
+        'url': 'https://economist.zoom.us/rec/play/dUk_CNBETmZ5VA2BwEl-jjakPpJ3M1pcfVYAPRsoIbEByGsLjUZtaa4yCATQuOL3der8BlTwxQePl_j0.EImBkXzTIaPvdZO5',
+        'md5': 'ab445e8c911fddc4f9adc842c2c5d434',
+        'info_dict': {
+            'id': 'dUk_CNBETmZ5VA2BwEl-jjakPpJ3M1pcfVYAPRsoIbEByGsLjUZtaa4yCATQuOL3der8BlTwxQePl_j0.EImBkXzTIaPvdZO5',
+            'ext': 'mp4',
+            'title': 'China\'s "two sessions" and the new five-year plan',
+        }
+    }
+
+    def _real_extract(self, url):
+        base_url, play_id = re.match(self._VALID_URL, url).groups()
+        webpage = self._download_webpage(url, play_id)
+
+        try:
+            form = self._form_hidden_inputs('password_form', webpage)
+        except ExtractorError:
+            form = None
+        if form:
+            password = self._downloader.params.get('videopassword')
+            if not password:
+                raise ExtractorError(
+                    'This video is protected by a passcode, use the --video-password option', expected=True)
+            is_meeting = form.get('useWhichPasswd') == 'meeting'
+            validation = self._download_json(
+                base_url + 'rec/validate%s_passwd' % ('_meet' if is_meeting else ''),
+                play_id, 'Validating passcode', 'Wrong passcode', data=urlencode_postdata({
+                    'id': form[('meet' if is_meeting else 'file') + 'Id'],
+                    'passwd': password,
+                    'action': form.get('action'),
+                }))
+            if not validation.get('status'):
+                raise ExtractorError(validation['errorMessage'], expected=True)
+            webpage = self._download_webpage(url, play_id)
+
+        data = self._parse_json(self._search_regex(
+            r'(?s)window\.__data__\s*=\s*({.+?});',
+            webpage, 'data'), play_id, js_to_json)
+
+        return {
+            'id': play_id,
+            'title': data['topic'],
+            'url': data['viewMp4Url'],
+            'width': int_or_none(data.get('viewResolvtionsWidth')),
+            'height': int_or_none(data.get('viewResolvtionsHeight')),
+            'http_headers': {
+                'Referer': base_url,
+            },
+            'filesize_approx': parse_filesize(data.get('fileSize')),
+        }

From c2fbfb49da2657002fafcf4c609f8d91030a6ef3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Mar 2021 00:03:00 +0700
Subject: [PATCH 290/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 21 +++++++++++++++++++++
 1 file changed, 21 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 73fe316b9..1c3313280 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,24 @@
+version <unreleased>
+
+Extractors
++ [zoom] Add support for zoom.us (#16597, #27002, #28531)
+* [bbc] Fix BBC IPlayer Episodes/Group extraction (#28360)
+* [youtube] Fix default value for youtube_include_dash_manifest (#28523)
+* [zingmp3] Fix extraction (#11589, #16409, #16968, #27205)
++ [vgtv] Add support for new tv.aftonbladet.se URL schema (#28514)
++ [tiktok] Detect private videos (#28453)
+* [vimeo:album] Fix extraction for albums with number of videos multiple
+  to page size (#28486)
+* [vvvvid] Fix kenc format extraction (#28473)
+* [mlb] Fix video extraction (#21241)
+* [svtplay] Improve extraction (#28448)
+* [applepodcasts] Fix extraction (#28445)
+* [rtve] Improve extraction
+    + Extract all formats
+    * Fix RTVE Infantil extraction (#24851)
+    + Extract is_live and series
+
+
 version 2021.03.14
 
 Core

From 76da1c954aebba4af8def73dd2319fab2e27e50a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 25 Mar 2021 00:04:10 +0700
Subject: [PATCH 291/860] release 2021.03.25

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 docs/supportedsites.md                           | 6 +++++-
 youtube_dl/version.py                            | 2 +-
 8 files changed, 19 insertions(+), 15 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 9cc79eff4..7feb0298c 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.14. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.25. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.03.14**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.25**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.03.14
+ [debug] youtube-dl version 2021.03.25
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index 3296e44a8..49e18173d 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.14. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.25. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.03.14**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.25**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index f74c29736..a1486b133 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.14. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.25. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.03.14**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.25**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index ae9e273ef..7eaf5a202 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.14. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.25. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.03.14**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.25**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.03.14
+ [debug] youtube-dl version 2021.03.25
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index 04fbea2f6..20042d98c 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.14. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.25. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.03.14**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.25**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index 1c3313280..1b49e411a 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.03.25
 
 Extractors
 + [zoom] Add support for zoom.us (#16597, #27002, #28531)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 2c00ec406..d2ad937a4 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -90,7 +90,8 @@
  - **bbc**: BBC
  - **bbc.co.uk**: BBC iPlayer
  - **bbc.co.uk:article**: BBC articles
- - **bbc.co.uk:iplayer:playlist**
+ - **bbc.co.uk:iplayer:episodes**
+ - **bbc.co.uk:iplayer:group**
  - **bbc.co.uk:playlist**
  - **BBVTV**
  - **Beatport**
@@ -522,6 +523,7 @@
  - **mixcloud:playlist**
  - **mixcloud:user**
  - **MLB**
+ - **MLBVideo**
  - **Mnet**
  - **MNetTV**
  - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net
@@ -1212,4 +1214,6 @@
  - **ZDFChannel**
  - **Zhihu**
  - **zingmp3**: mp3.zing.vn
+ - **zingmp3:album**
+ - **zoom**
  - **Zype**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 5a540119c..e87b820fa 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.03.14'
+__version__ = '2021.03.25'

From 8562218350a79d4709da8593bb0c538aa0824acf Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 24 Mar 2021 19:28:51 +0100
Subject: [PATCH 292/860] [ard] improve clip id extraction(#22724)(closes
 #28528)

---
 youtube_dl/extractor/ard.py | 23 +++++++++++------------
 1 file changed, 11 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index 143fc51e9..d57c5ba0f 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -335,7 +335,7 @@ class ARDIE(InfoExtractor):
 
 
 class ARDBetaMediathekIE(ARDMediathekBaseIE):
-    _VALID_URL = r'https://(?:(?:beta|www)\.)?ardmediathek\.de/(?P<client>[^/]+)/(?:player|live|video)/(?P<display_id>(?:[^/]+/)*)(?P<video_id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https://(?:(?:beta|www)\.)?ardmediathek\.de/(?:[^/]+/)?(?:player|live|video)/(?:[^/]+/)*(?P<id>Y3JpZDovL[a-zA-Z0-9]+)'
     _TESTS = [{
         'url': 'https://www.ardmediathek.de/mdr/video/die-robuste-roswita/Y3JpZDovL21kci5kZS9iZWl0cmFnL2Ntcy84MWMxN2MzZC0wMjkxLTRmMzUtODk4ZS0wYzhlOWQxODE2NGI/',
         'md5': 'a1dc75a39c61601b980648f7c9f9f71d',
@@ -365,22 +365,22 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
     }, {
         'url': 'https://www.ardmediathek.de/swr/live/Y3JpZDovL3N3ci5kZS8xMzQ4MTA0Mg',
         'only_matching': True,
+    }, {
+        'url': 'https://www.ardmediathek.de/video/coronavirus-update-ndr-info/astrazeneca-kurz-lockdown-und-pims-syndrom-81/ndr/Y3JpZDovL25kci5kZS84NzE0M2FjNi0wMWEwLTQ5ODEtOTE5NS1mOGZhNzdhOTFmOTI/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.ardmediathek.de/ard/player/Y3JpZDovL3dkci5kZS9CZWl0cmFnLWQ2NDJjYWEzLTMwZWYtNGI4NS1iMTI2LTU1N2UxYTcxOGIzOQ/tatort-duo-koeln-leipzig-ihr-kinderlein-kommet',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('video_id')
-        display_id = mobj.group('display_id')
-        if display_id:
-            display_id = display_id.rstrip('/')
-        if not display_id:
-            display_id = video_id
+        video_id = self._match_id(url)
 
         player_page = self._download_json(
             'https://api.ardmediathek.de/public-gateway',
-            display_id, data=json.dumps({
+            video_id, data=json.dumps({
                 'query': '''{
-  playerPage(client:"%s", clipId: "%s") {
+  playerPage(client: "ard", clipId: "%s") {
     blockedByFsk
     broadcastedOn
     maturityContentRating
@@ -410,7 +410,7 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
       }
     }
   }
-}''' % (mobj.group('client'), video_id),
+}''' % video_id,
             }).encode(), headers={
                 'Content-Type': 'application/json'
             })['data']['playerPage']
@@ -435,7 +435,6 @@ class ARDBetaMediathekIE(ARDMediathekBaseIE):
                 r'\(FSK\s*(\d+)\)\s*$', description, 'age limit', default=None))
         info.update({
             'age_limit': age_limit,
-            'display_id': display_id,
             'title': title,
             'description': description,
             'timestamp': unified_timestamp(player_page.get('broadcastedOn')),

From 605e7b5e47c60c3ed7c2ca71df4d6bbd49fa8a77 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 25 Mar 2021 12:53:18 +0100
Subject: [PATCH 293/860] [youtube:tab] fix playlist/comunity continuation
 items extraction(closes #28266)

---
 youtube_dl/extractor/youtube.py | 13 +++++++++++--
 1 file changed, 11 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index badca3977..faf3a344e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -24,6 +24,7 @@ from ..jsinterp import JSInterpreter
 from ..utils import (
     ExtractorError,
     clean_html,
+    dict_get,
     float_or_none,
     int_or_none,
     mimetype2ext,
@@ -2541,13 +2542,14 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                     continuation = self._extract_continuation(continuation_renderer)
                     continue
 
+            on_response_received = dict_get(response, ('onResponseReceivedActions', 'onResponseReceivedEndpoints'))
             continuation_items = try_get(
-                response, lambda x: x['onResponseReceivedActions'][0]['appendContinuationItemsAction']['continuationItems'], list)
+                on_response_received, lambda x: x[0]['appendContinuationItemsAction']['continuationItems'], list)
             if continuation_items:
                 continuation_item = continuation_items[0]
                 if not isinstance(continuation_item, dict):
                     continue
-                renderer = continuation_item.get('gridVideoRenderer')
+                renderer = self._extract_grid_item_renderer(continuation_item)
                 if renderer:
                     grid_renderer = {'items': continuation_items}
                     for entry in self._grid_entries(grid_renderer):
@@ -2561,6 +2563,13 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                         yield entry
                     continuation = self._extract_continuation(video_list_renderer)
                     continue
+                renderer = continuation_item.get('backstagePostThreadRenderer')
+                if renderer:
+                    continuation_renderer = {'contents': continuation_items}
+                    for entry in self._post_thread_continuation_entries(continuation_renderer):
+                        yield entry
+                    continuation = self._extract_continuation(continuation_renderer)
+                    continue
 
             break
 

From c78591187080e7316c0042309fe956bfd0d38d30 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 25 Mar 2021 17:06:57 +0100
Subject: [PATCH 294/860] [vimeo] fix unlisted video extraction(closes #28414)

---
 youtube_dl/extractor/vimeo.py | 39 ++++++++++++++++++++++++++++++-----
 1 file changed, 34 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 955651bec..5800962ea 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -24,6 +24,7 @@ from ..utils import (
     merge_dicts,
     OnDemandPagedList,
     parse_filesize,
+    parse_iso8601,
     RegexNotFoundError,
     sanitized_Request,
     smuggle_url,
@@ -278,7 +279,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                             )?
                         (?:videos?/)?
                         (?P<id>[0-9]+)
-                        (?:/[\da-f]+)?
+                        (?:/(?P<unlisted_hash>[\da-f]{10}))?
                         /?(?:[?&].*)?(?:[#].*)?$
                     '''
     IE_NAME = 'vimeo'
@@ -577,11 +578,37 @@ class VimeoIE(VimeoBaseInfoExtractor):
         if 'Referer' not in headers:
             headers['Referer'] = url
 
-        channel_id = self._search_regex(
-            r'vimeo\.com/channels/([^/]+)', url, 'channel id', default=None)
-
         # Extract ID from URL
-        video_id = self._match_id(url)
+        video_id, unlisted_hash = re.match(self._VALID_URL, url).groups()
+        if unlisted_hash:
+            token = self._download_json(
+                'https://vimeo.com/_rv/jwt', video_id, headers={
+                    'X-Requested-With': 'XMLHttpRequest'
+                })['token']
+            video = self._download_json(
+                'https://api.vimeo.com/videos/%s:%s' % (video_id, unlisted_hash),
+                video_id, headers={
+                    'Authorization': 'jwt ' + token,
+                }, query={
+                    'fields': 'config_url,created_time,description,license,metadata.connections.comments.total,metadata.connections.likes.total,release_time,stats.plays',
+                })
+            info = self._parse_config(self._download_json(
+                video['config_url'], video_id), video_id)
+            self._vimeo_sort_formats(info['formats'])
+            get_timestamp = lambda x: parse_iso8601(video.get(x + '_time'))
+            info.update({
+                'description': video.get('description'),
+                'license': video.get('license'),
+                'release_timestamp': get_timestamp('release'),
+                'timestamp': get_timestamp('created'),
+                'view_count': int_or_none(try_get(video, lambda x: x['stats']['plays'])),
+            })
+            connections = try_get(
+                video, lambda x: x['metadata']['connections'], dict) or {}
+            for k in ('comment', 'like'):
+                info[k + '_count'] = int_or_none(try_get(connections, lambda x: x[k + 's']['total']))
+            return info
+
         orig_url = url
         is_pro = 'vimeopro.com/' in url
         is_player = '://player.vimeo.com/video/' in url
@@ -756,6 +783,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 r'<link[^>]+rel=["\']license["\'][^>]+href=(["\'])(?P<license>(?:(?!\1).)+)\1',
                 webpage, 'license', default=None, group='license')
 
+        channel_id = self._search_regex(
+            r'vimeo\.com/channels/([^/]+)', url, 'channel id', default=None)
         channel_url = 'https://vimeo.com/channels/%s' % channel_id if channel_id else None
 
         info_dict = {

From cc777dcaa0f3331626f33a7e4c61d804c43f4b5c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 27 Mar 2021 17:37:45 +0100
Subject: [PATCH 295/860] [picarto] fix live stream extraction(closes #28532)

---
 youtube_dl/extractor/picarto.py | 96 ++++++++++++---------------------
 1 file changed, 35 insertions(+), 61 deletions(-)

diff --git a/youtube_dl/extractor/picarto.py b/youtube_dl/extractor/picarto.py
index 8099ef1d6..e6c51e16b 100644
--- a/youtube_dl/extractor/picarto.py
+++ b/youtube_dl/extractor/picarto.py
@@ -1,22 +1,15 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-import time
-
 from .common import InfoExtractor
-from ..compat import compat_str
 from ..utils import (
     ExtractorError,
     js_to_json,
-    try_get,
-    update_url_query,
-    urlencode_postdata,
 )
 
 
 class PicartoIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www.)?picarto\.tv/(?P<id>[a-zA-Z0-9]+)(?:/(?P<token>[a-zA-Z0-9]+))?'
+    _VALID_URL = r'https?://(?:www.)?picarto\.tv/(?P<id>[a-zA-Z0-9]+)'
     _TEST = {
         'url': 'https://picarto.tv/Setz',
         'info_dict': {
@@ -34,65 +27,46 @@ class PicartoIE(InfoExtractor):
         return False if PicartoVodIE.suitable(url) else super(PicartoIE, cls).suitable(url)
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        channel_id = mobj.group('id')
+        channel_id = self._match_id(url)
 
-        metadata = self._download_json(
-            'https://api.picarto.tv/v1/channel/name/' + channel_id,
-            channel_id)
+        data = self._download_json(
+            'https://ptvintern.picarto.tv/ptvapi', channel_id, query={
+                'query': '''{
+  channel(name: "%s") {
+    adult
+    id
+    online
+    stream_name
+    title
+  }
+  getLoadBalancerUrl(channel_name: "%s") {
+    url
+  }
+}''' % (channel_id, channel_id),
+            })['data']
+        metadata = data['channel']
 
-        if metadata.get('online') is False:
+        if metadata.get('online') == 0:
             raise ExtractorError('Stream is offline', expected=True)
+        title = metadata['title']
 
         cdn_data = self._download_json(
-            'https://picarto.tv/process/channel', channel_id,
-            data=urlencode_postdata({'loadbalancinginfo': channel_id}),
-            note='Downloading load balancing info')
+            data['getLoadBalancerUrl']['url'] + '/stream/json_' + metadata['stream_name'] + '.js',
+            channel_id, 'Downloading load balancing info')
 
-        token = mobj.group('token') or 'public'
-        params = {
-            'con': int(time.time() * 1000),
-            'token': token,
-        }
-
-        prefered_edge = cdn_data.get('preferedEdge')
         formats = []
-
-        for edge in cdn_data['edges']:
-            edge_ep = edge.get('ep')
-            if not edge_ep or not isinstance(edge_ep, compat_str):
+        for source in (cdn_data.get('source') or []):
+            source_url = source.get('url')
+            if not source_url:
                 continue
-            edge_id = edge.get('id')
-            for tech in cdn_data['techs']:
-                tech_label = tech.get('label')
-                tech_type = tech.get('type')
-                preference = 0
-                if edge_id == prefered_edge:
-                    preference += 1
-                format_id = []
-                if edge_id:
-                    format_id.append(edge_id)
-                if tech_type == 'application/x-mpegurl' or tech_label == 'HLS':
-                    format_id.append('hls')
-                    formats.extend(self._extract_m3u8_formats(
-                        update_url_query(
-                            'https://%s/hls/%s/index.m3u8'
-                            % (edge_ep, channel_id), params),
-                        channel_id, 'mp4', preference=preference,
-                        m3u8_id='-'.join(format_id), fatal=False))
-                    continue
-                elif tech_type == 'video/mp4' or tech_label == 'MP4':
-                    format_id.append('mp4')
-                    formats.append({
-                        'url': update_url_query(
-                            'https://%s/mp4/%s.mp4' % (edge_ep, channel_id),
-                            params),
-                        'format_id': '-'.join(format_id),
-                        'preference': preference,
-                    })
-                else:
-                    # rtmp format does not seem to work
-                    continue
+            source_type = source.get('type')
+            if source_type == 'html5/application/vnd.apple.mpegurl':
+                formats.extend(self._extract_m3u8_formats(
+                    source_url, channel_id, 'mp4', m3u8_id='hls', fatal=False))
+            elif source_type == 'html5/video/mp4':
+                formats.append({
+                    'url': source_url,
+                })
         self._sort_formats(formats)
 
         mature = metadata.get('adult')
@@ -103,10 +77,10 @@ class PicartoIE(InfoExtractor):
 
         return {
             'id': channel_id,
-            'title': self._live_title(metadata.get('title') or channel_id),
+            'title': self._live_title(title.strip()),
             'is_live': True,
-            'thumbnail': try_get(metadata, lambda x: x['thumbnails']['web']),
             'channel': channel_id,
+            'channel_id': metadata.get('id'),
             'channel_url': 'https://picarto.tv/%s' % channel_id,
             'age_limit': age_limit,
             'formats': formats,

From 49fc0a567febda65709cc5154ff046684a3b8427 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 27 Mar 2021 19:11:41 +0100
Subject: [PATCH 296/860] [youtube] fix video's channel extraction(closes
 #28562)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index faf3a344e..e48c5a7d2 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1896,7 +1896,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     info['channel'] = get_text(try_get(
                         vsir,
                         lambda x: x['owner']['videoOwnerRenderer']['title'],
-                        compat_str))
+                        dict))
                     rows = try_get(
                         vsir,
                         lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],

From 87a8bde7775ebc31175ebb111015b4052b50b7db Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 28 Mar 2021 08:46:33 +0100
Subject: [PATCH 297/860] [sbs] add support for ondemand watch URLs(closes
 #28566)

---
 youtube_dl/extractor/sbs.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/sbs.py b/youtube_dl/extractor/sbs.py
index f722528cd..0a806ee4e 100644
--- a/youtube_dl/extractor/sbs.py
+++ b/youtube_dl/extractor/sbs.py
@@ -10,7 +10,7 @@ from ..utils import (
 
 class SBSIE(InfoExtractor):
     IE_DESC = 'sbs.com.au'
-    _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/(?:ondemand(?:/video/(?:single/)?|.*?\bplay=)|news/(?:embeds/)?video/)(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?sbs\.com\.au/(?:ondemand(?:/video/(?:single/)?|.*?\bplay=|/watch/)|news/(?:embeds/)?video/)(?P<id>[0-9]+)'
 
     _TESTS = [{
         # Original URL is handled by the generic IE which finds the iframe:
@@ -43,6 +43,9 @@ class SBSIE(InfoExtractor):
     }, {
         'url': 'https://www.sbs.com.au/news/embeds/video/1840778819866',
         'only_matching': True,
+    }, {
+        'url': 'https://www.sbs.com.au/ondemand/watch/1698704451971',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From da762c4e329f6158956ddd51ac294e9183e5ce89 Mon Sep 17 00:00:00 2001
From: Chris Hranj <christopher.hranj@gmail.com>
Date: Mon, 29 Mar 2021 15:05:19 -0400
Subject: [PATCH 298/860] [instagram] Improve title extraction and extract
 duration (#28469)

Co-authored-by: Sergey M. <dstftw@gmail.com>
---
 youtube_dl/extractor/instagram.py | 29 +++++++++++++++++++++++++++--
 1 file changed, 27 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/instagram.py b/youtube_dl/extractor/instagram.py
index 1eeddc3b6..12e10143c 100644
--- a/youtube_dl/extractor/instagram.py
+++ b/youtube_dl/extractor/instagram.py
@@ -12,6 +12,7 @@ from ..compat import (
 )
 from ..utils import (
     ExtractorError,
+    float_or_none,
     get_element_by_attribute,
     int_or_none,
     lowercase_escape,
@@ -32,6 +33,7 @@ class InstagramIE(InfoExtractor):
             'title': 'Video by naomipq',
             'description': 'md5:1f17f0ab29bd6fe2bfad705f58de3cb8',
             'thumbnail': r're:^https?://.*\.jpg',
+            'duration': 0,
             'timestamp': 1371748545,
             'upload_date': '20130620',
             'uploader_id': 'naomipq',
@@ -48,6 +50,7 @@ class InstagramIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Video by britneyspears',
             'thumbnail': r're:^https?://.*\.jpg',
+            'duration': 0,
             'timestamp': 1453760977,
             'upload_date': '20160125',
             'uploader_id': 'britneyspears',
@@ -86,6 +89,24 @@ class InstagramIE(InfoExtractor):
             'title': 'Post by instagram',
             'description': 'md5:0f9203fc6a2ce4d228da5754bcf54957',
         },
+    }, {
+        # IGTV
+        'url': 'https://www.instagram.com/tv/BkfuX9UB-eK/',
+        'info_dict': {
+            'id': 'BkfuX9UB-eK',
+            'ext': 'mp4',
+            'title': 'Fingerboarding Tricks with @cass.fb',
+            'thumbnail': r're:^https?://.*\.jpg',
+            'duration': 53.83,
+            'timestamp': 1530032919,
+            'upload_date': '20180626',
+            'uploader_id': 'instagram',
+            'uploader': 'Instagram',
+            'like_count': int,
+            'comment_count': int,
+            'comments': list,
+            'description': 'Meet Cass Hirst (@cass.fb), a fingerboarding pro who can perform tiny ollies and kickflips while blindfolded.',
+        }
     }, {
         'url': 'https://instagram.com/p/-Cmh1cukG2/',
         'only_matching': True,
@@ -159,7 +180,9 @@ class InstagramIE(InfoExtractor):
             description = try_get(
                 media, lambda x: x['edge_media_to_caption']['edges'][0]['node']['text'],
                 compat_str) or media.get('caption')
+            title = media.get('title')
             thumbnail = media.get('display_src') or media.get('display_url')
+            duration = float_or_none(media.get('video_duration'))
             timestamp = int_or_none(media.get('taken_at_timestamp') or media.get('date'))
             uploader = media.get('owner', {}).get('full_name')
             uploader_id = media.get('owner', {}).get('username')
@@ -200,9 +223,10 @@ class InstagramIE(InfoExtractor):
                             continue
                         entries.append({
                             'id': node.get('shortcode') or node['id'],
-                            'title': 'Video %d' % edge_num,
+                            'title': node.get('title') or 'Video %d' % edge_num,
                             'url': node_video_url,
                             'thumbnail': node.get('display_url'),
+                            'duration': float_or_none(node.get('video_duration')),
                             'width': int_or_none(try_get(node, lambda x: x['dimensions']['width'])),
                             'height': int_or_none(try_get(node, lambda x: x['dimensions']['height'])),
                             'view_count': int_or_none(node.get('video_view_count')),
@@ -239,8 +263,9 @@ class InstagramIE(InfoExtractor):
             'id': video_id,
             'formats': formats,
             'ext': 'mp4',
-            'title': 'Video by %s' % uploader_id,
+            'title': title or 'Video by %s' % uploader_id,
             'description': description,
+            'duration': duration,
             'thumbnail': thumbnail,
             'timestamp': timestamp,
             'uploader_id': uploader_id,

From 287e50b56b4c71da8fd0c3ffdeca9bff5ab0b005 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 30 Mar 2021 03:37:43 +0700
Subject: [PATCH 299/860] [francetvinfo] Improve video id extraction (closes
 #28584)

---
 youtube_dl/extractor/francetv.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 3ca415077..7cc88bf18 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -399,7 +399,8 @@ class FranceTVInfoIE(FranceTVBaseInfoExtractor):
         video_id = self._search_regex(
             (r'player\.load[^;]+src:\s*["\']([^"\']+)',
              r'id-video=([^@]+@[^"]+)',
-             r'<a[^>]+href="(?:https?:)?//videos\.francetv\.fr/video/([^@]+@[^"]+)"'),
+             r'<a[^>]+href="(?:https?:)?//videos\.francetv\.fr/video/([^@]+@[^"]+)"',
+             r'data-id=["\']([\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'),
             webpage, 'video id')
 
         return self._make_url_result(video_id)

From 955894e72fd8d4fdce5d85fc006d548278e6d9eb Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 30 Mar 2021 10:00:40 +0100
Subject: [PATCH 300/860] [vlive] fix inkey request(closes #28589)

---
 youtube_dl/extractor/vlive.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vlive.py b/youtube_dl/extractor/vlive.py
index e2f5d81b8..42da34d44 100644
--- a/youtube_dl/extractor/vlive.py
+++ b/youtube_dl/extractor/vlive.py
@@ -106,7 +106,7 @@ class VLiveIE(VLiveBaseIE):
             raise ExtractorError('Unable to log in', expected=True)
 
     def _call_api(self, path_template, video_id, fields=None):
-        query = {'appId': self._APP_ID, 'gcc': 'KR'}
+        query = {'appId': self._APP_ID, 'gcc': 'KR', 'platformType': 'PC'}
         if fields:
             query['fields'] = fields
         try:

From 207bc35d348efdfdfe2bd7119e004a1acf0ab3d2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 31 Mar 2021 02:58:01 +0700
Subject: [PATCH 301/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 1b49e411a..1297c19f7 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,18 @@
+version <unreleased>
+
+Extractors
+* [vlive] Fix inkey request (#28589)
+* [francetvinfo] Improve video id extraction (#28584)
++ [instagram] Extract duration (#28469)
+* [instagram] Improve title extraction (#28469)
++ [sbs] Add support for ondemand watch URLs (#28566)
+* [youtube] Fix video's channel extraction (#28562)
+* [picarto] Fix live stream extraction (#28532)
+* [vimeo] Fix unlisted video extraction (#28414)
+* [youtube:tab] Fix playlist/community continuation items extraction (#28266)
+* [ard] Improve clip id extraction (#22724, #28528)
+
+
 version 2021.03.25
 
 Extractors

From 8f493de9fb3a7f123bdf887163efa06ce9d6b051 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 31 Mar 2021 02:59:07 +0700
Subject: [PATCH 302/860] release 2021.03.31

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 youtube_dl/version.py                            | 2 +-
 7 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 7feb0298c..2ac4df8db 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.25. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.03.25**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.31**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.03.25
+ [debug] youtube-dl version 2021.03.31
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index 49e18173d..5ad5590bf 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.25. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.03.25**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.31**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index a1486b133..ea96c4c20 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.25. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.03.25**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.31**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index 7eaf5a202..ed3abd45c 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.25. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.03.25**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.31**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.03.25
+ [debug] youtube-dl version 2021.03.31
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index 20042d98c..c1067ee1f 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.25. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.03.25**
+- [ ] I've verified that I'm running youtube-dl version **2021.03.31**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index 1297c19f7..4c094b771 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.03.31
 
 Extractors
 * [vlive] Fix inkey request (#28589)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index e87b820fa..bcfdae23d 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.03.25'
+__version__ = '2021.03.31'

From 28bab774a0df2c80b689c277390da8617131db35 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Tue, 30 Mar 2021 21:44:41 +0100
Subject: [PATCH 303/860] [youtube] imporve age-restricted video
 extraction(#28578)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e48c5a7d2..6a92938a5 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1432,7 +1432,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         base_url = self.http_scheme() + '//www.youtube.com/'
         webpage_url = base_url + 'watch?v=' + video_id
         webpage = self._download_webpage(
-            webpage_url + '&bpctr=9999999999', video_id, fatal=False)
+            webpage_url + '&bpctr=9999999999&has_verified=1', video_id, fatal=False)
 
         player_response = None
         if webpage:

From b97fb2edac25182ff3dcf4cb8537517a1ec9e4de Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 31 Mar 2021 20:07:13 +0100
Subject: [PATCH 304/860] [vimeo] fix password protected review
 extraction(closes #27591)

---
 youtube_dl/extractor/vimeo.py | 64 ++++++++++++++++++++++-------------
 1 file changed, 41 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 5800962ea..a90cf0630 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -75,25 +75,28 @@ class VimeoBaseInfoExtractor(InfoExtractor):
                     expected=True)
             raise ExtractorError('Unable to log in')
 
-    def _verify_video_password(self, url, video_id, webpage):
+    def _get_video_password(self):
         password = self._downloader.params.get('videopassword')
         if password is None:
-            raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
-        token, vuid = self._extract_xsrft_and_vuid(webpage)
-        data = urlencode_postdata({
-            'password': password,
-            'token': token,
-        })
+            raise ExtractorError(
+                'This video is protected by a password, use the --video-password option',
+                expected=True)
+        return password
+
+    def _verify_video_password(self, url, video_id, password, token, vuid):
         if url.startswith('http://'):
             # vimeo only supports https now, but the user can give an http url
             url = url.replace('http://', 'https://')
-        password_request = sanitized_Request(url + '/password', data)
-        password_request.add_header('Content-Type', 'application/x-www-form-urlencoded')
-        password_request.add_header('Referer', url)
         self._set_vimeo_cookie('vuid', vuid)
         return self._download_webpage(
-            password_request, video_id,
-            'Verifying the password', 'Wrong password')
+            url + '/password', video_id, 'Verifying the password',
+            'Wrong password', data=urlencode_postdata({
+                'password': password,
+                'token': token,
+            }), headers={
+                'Content-Type': 'application/x-www-form-urlencoded',
+                'Referer': url,
+            })
 
     def _extract_xsrft_and_vuid(self, webpage):
         xsrft = self._search_regex(
@@ -332,9 +335,9 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'id': '54469442',
                 'ext': 'mp4',
                 'title': 'Kathy Sierra: Building the minimum Badass User, Business of Software 2012',
-                'uploader': 'The BLN & Business of Software',
-                'uploader_url': r're:https?://(?:www\.)?vimeo\.com/theblnbusinessofsoftware',
-                'uploader_id': 'theblnbusinessofsoftware',
+                'uploader': 'Business of Software',
+                'uploader_url': r're:https?://(?:www\.)?vimeo\.com/businessofsoftware',
+                'uploader_id': 'businessofsoftware',
                 'duration': 3610,
                 'description': None,
             },
@@ -469,6 +472,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'skip_download': True,
             },
             'expected_warnings': ['Unable to download JSON metadata'],
+            'skip': 'this page is no longer available.',
         },
         {
             'url': 'http://player.vimeo.com/video/68375962',
@@ -551,9 +555,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
         return urls[0] if urls else None
 
     def _verify_player_video_password(self, url, video_id, headers):
-        password = self._downloader.params.get('videopassword')
-        if password is None:
-            raise ExtractorError('This video is protected by a password, use the --video-password option', expected=True)
+        password = self._get_video_password()
         data = urlencode_postdata({
             'password': base64.b64encode(password.encode()),
         })
@@ -697,7 +699,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
             if re.search(r'<form[^>]+?id="pw_form"', webpage) is not None:
                 if '_video_password_verified' in data:
                     raise ExtractorError('video password verification failed!')
-                self._verify_video_password(redirect_url, video_id, webpage)
+                video_password = self._get_video_password()
+                token, vuid = self._extract_xsrft_and_vuid(webpage)
+                self._verify_video_password(
+                    redirect_url, video_id, video_password, token, vuid)
                 return self._real_extract(
                     smuggle_url(redirect_url, {'_video_password_verified': 'verified'}))
             else:
@@ -1091,10 +1096,23 @@ class VimeoReviewIE(VimeoBaseInfoExtractor):
 
     def _real_extract(self, url):
         page_url, video_id = re.match(self._VALID_URL, url).groups()
-        clip_data = self._download_json(
-            page_url.replace('/review/', '/review/data/'),
-            video_id)['clipData']
-        config_url = clip_data['configUrl']
+        data = self._download_json(
+            page_url.replace('/review/', '/review/data/'), video_id)
+        if data.get('isLocked') is True:
+            video_password = self._get_video_password()
+            viewer = self._download_json(
+                'https://vimeo.com/_rv/viewer', video_id)
+            webpage = self._verify_video_password(
+                'https://vimeo.com/' + video_id, video_id,
+                video_password, viewer['xsrft'], viewer['vuid'])
+            clip_page_config = self._parse_json(self._search_regex(
+                r'window\.vimeo\.clip_page_config\s*=\s*({.+?});',
+                webpage, 'clip page config'), video_id)
+            config_url = clip_page_config['player']['config_url']
+            clip_data = clip_page_config.get('clip') or {}
+        else:
+            clip_data = data['clipData']
+            config_url = clip_data['configUrl']
         config = self._download_json(config_url, video_id)
         info_dict = self._parse_config(config, video_id)
         source_format = self._extract_original_format(

From 14f29f087e6097feb46bdb84878924bc410a57eb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Apr 2021 04:05:10 +0700
Subject: [PATCH 305/860] [youtube] Setup CONSENT cookie when needed (closes
 #28604)

---
 youtube_dl/extractor/youtube.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6a92938a5..b940c0bad 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -249,7 +249,23 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
         return True
 
+    def _initialize_consent(self):
+        cookies = self._get_cookies('https://www.youtube.com/')
+        if cookies.get('__Secure-3PSID'):
+            return
+        consent_id = None
+        consent = cookies.get('CONSENT')
+        if consent:
+            if 'YES' in consent.value:
+                return
+            consent_id = self._search_regex(
+                r'PENDING\+(\d+)', consent.value, 'consent', default=None)
+        if not consent_id:
+            consent_id = random.randint(100, 999)
+        self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
+
     def _real_initialize(self):
+        self._initialize_consent()
         if self._downloader is None:
             return
         if not self._login():

From e789bb1aa4cb627d3d7ca79a5e5daa8d2f58cda6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Apr 2021 04:43:08 +0700
Subject: [PATCH 306/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 4c094b771..ee2dc88bc 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,11 @@
+version <unreleased>
+
+Extractors
+* [youtube] Setup CONSENT cookie when needed (#28604)
+* [vimeo] Fix password protected review extraction (#27591)
+* [youtube] Improve age-restricted video extraction (#28578)
+
+
 version 2021.03.31
 
 Extractors

From ca304beb1538e54c5a18fdd50846ed2259d63b8c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 1 Apr 2021 04:47:11 +0700
Subject: [PATCH 307/860] release 2021.04.01

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 youtube_dl/version.py                            | 2 +-
 7 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 2ac4df8db..98ec799e8 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.01. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.03.31**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.01**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.03.31
+ [debug] youtube-dl version 2021.04.01
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index 5ad5590bf..5387a6cd1 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.01. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.03.31**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.01**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index ea96c4c20..945c80366 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.01. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.03.31**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.01**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index ed3abd45c..0acc8b679 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.01. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.03.31**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.01**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.03.31
+ [debug] youtube-dl version 2021.04.01
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index c1067ee1f..42c3126a3 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.03.31. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.01. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.03.31**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.01**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index ee2dc88bc..4304ecd9e 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.04.01
 
 Extractors
 * [youtube] Setup CONSENT cookie when needed (#28604)
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index bcfdae23d..0457d1a15 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.03.31'
+__version__ = '2021.04.01'

From 37488630703944b4f2bda84a26391ae61d29e15b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 1 Apr 2021 11:50:30 +0100
Subject: [PATCH 308/860] [youtube:tab] Add support for hashtag videos
 extraction(closes #28308)

---
 youtube_dl/extractor/youtube.py | 137 ++++++++++++++++++++------------
 1 file changed, 84 insertions(+), 53 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b940c0bad..1f5497e24 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1959,7 +1959,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                             invidio\.us
                         )/
                         (?:
-                            (?:channel|c|user|feed)/|
+                            (?:channel|c|user|feed|hashtag)/|
                             (?:playlist|watch)\?.*?\blist=|
                             (?!(?:watch|embed|v|e)\b)
                         )
@@ -2245,6 +2245,13 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
     }, {
         'url': 'https://www.youtube.com/TheYoungTurks/live',
         'only_matching': True,
+    }, {
+        'url': 'https://www.youtube.com/hashtag/cctv9',
+        'info_dict': {
+            'id': 'cctv9',
+            'title': '#cctv9',
+        },
+        'playlist_mincount': 350,
     }]
 
     @classmethod
@@ -2392,6 +2399,14 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             for entry in self._post_thread_entries(renderer):
                 yield entry
 
+    def _rich_grid_entries(self, contents):
+        for content in contents:
+            video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
+            if video_renderer:
+                entry = self._video_entry(video_renderer)
+                if entry:
+                    yield entry
+
     @staticmethod
     def _build_continuation_query(continuation, ctp=None):
         query = {
@@ -2442,55 +2457,60 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
         if not tab_content:
             return
         slr_renderer = try_get(tab_content, lambda x: x['sectionListRenderer'], dict)
-        if not slr_renderer:
-            return
-        is_channels_tab = tab.get('title') == 'Channels'
-        continuation = None
-        slr_contents = try_get(slr_renderer, lambda x: x['contents'], list) or []
-        for slr_content in slr_contents:
-            if not isinstance(slr_content, dict):
-                continue
-            is_renderer = try_get(slr_content, lambda x: x['itemSectionRenderer'], dict)
-            if not is_renderer:
-                continue
-            isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
-            for isr_content in isr_contents:
-                if not isinstance(isr_content, dict):
+        if slr_renderer:
+            is_channels_tab = tab.get('title') == 'Channels'
+            continuation = None
+            slr_contents = try_get(slr_renderer, lambda x: x['contents'], list) or []
+            for slr_content in slr_contents:
+                if not isinstance(slr_content, dict):
                     continue
-                renderer = isr_content.get('playlistVideoListRenderer')
-                if renderer:
-                    for entry in self._playlist_entries(renderer):
-                        yield entry
-                    continuation = self._extract_continuation(renderer)
+                is_renderer = try_get(slr_content, lambda x: x['itemSectionRenderer'], dict)
+                if not is_renderer:
                     continue
-                renderer = isr_content.get('gridRenderer')
-                if renderer:
-                    for entry in self._grid_entries(renderer):
-                        yield entry
-                    continuation = self._extract_continuation(renderer)
-                    continue
-                renderer = isr_content.get('shelfRenderer')
-                if renderer:
-                    for entry in self._shelf_entries(renderer, not is_channels_tab):
-                        yield entry
-                    continue
-                renderer = isr_content.get('backstagePostThreadRenderer')
-                if renderer:
-                    for entry in self._post_thread_entries(renderer):
-                        yield entry
-                    continuation = self._extract_continuation(renderer)
-                    continue
-                renderer = isr_content.get('videoRenderer')
-                if renderer:
-                    entry = self._video_entry(renderer)
-                    if entry:
-                        yield entry
+                isr_contents = try_get(is_renderer, lambda x: x['contents'], list) or []
+                for isr_content in isr_contents:
+                    if not isinstance(isr_content, dict):
+                        continue
+                    renderer = isr_content.get('playlistVideoListRenderer')
+                    if renderer:
+                        for entry in self._playlist_entries(renderer):
+                            yield entry
+                        continuation = self._extract_continuation(renderer)
+                        continue
+                    renderer = isr_content.get('gridRenderer')
+                    if renderer:
+                        for entry in self._grid_entries(renderer):
+                            yield entry
+                        continuation = self._extract_continuation(renderer)
+                        continue
+                    renderer = isr_content.get('shelfRenderer')
+                    if renderer:
+                        for entry in self._shelf_entries(renderer, not is_channels_tab):
+                            yield entry
+                        continue
+                    renderer = isr_content.get('backstagePostThreadRenderer')
+                    if renderer:
+                        for entry in self._post_thread_entries(renderer):
+                            yield entry
+                        continuation = self._extract_continuation(renderer)
+                        continue
+                    renderer = isr_content.get('videoRenderer')
+                    if renderer:
+                        entry = self._video_entry(renderer)
+                        if entry:
+                            yield entry
 
+                if not continuation:
+                    continuation = self._extract_continuation(is_renderer)
             if not continuation:
-                continuation = self._extract_continuation(is_renderer)
-
-        if not continuation:
-            continuation = self._extract_continuation(slr_renderer)
+                continuation = self._extract_continuation(slr_renderer)
+        else:
+            rich_grid_renderer = tab_content.get('richGridRenderer')
+            if not rich_grid_renderer:
+                return
+            for entry in self._rich_grid_entries(rich_grid_renderer.get('contents') or []):
+                yield entry
+            continuation = self._extract_continuation(rich_grid_renderer)
 
         headers = {
             'x-youtube-client-name': '1',
@@ -2586,6 +2606,12 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                         yield entry
                     continuation = self._extract_continuation(continuation_renderer)
                     continue
+                renderer = continuation_item.get('richItemRenderer')
+                if renderer:
+                    for entry in self._rich_grid_entries(continuation_items):
+                        yield entry
+                    continuation = self._extract_continuation({'contents': continuation_items})
+                    continue
 
             break
 
@@ -2642,7 +2668,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
         selected_tab = self._extract_selected_tab(tabs)
         renderer = try_get(
             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
-        playlist_id = title = description = None
+        playlist_id = item_id
+        title = description = None
         if renderer:
             channel_title = renderer.get('title') or item_id
             tab_title = selected_tab.get('title')
@@ -2651,12 +2678,16 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                 title += ' - %s' % tab_title
             description = renderer.get('description')
             playlist_id = renderer.get('externalId')
-        renderer = try_get(
-            data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
-        if renderer:
-            title = renderer.get('title')
-            description = None
-            playlist_id = item_id
+        else:
+            renderer = try_get(
+                data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
+            if renderer:
+                title = renderer.get('title')
+            else:
+                renderer = try_get(
+                    data, lambda x: x['header']['hashtagHeaderRenderer'], dict)
+                if renderer:
+                    title = try_get(renderer, lambda x: x['hashtag']['simpleText'])
         playlist = self.playlist_result(
             self._entries(selected_tab, identity_token),
             playlist_id=playlist_id, playlist_title=title,

From c5aa8f36bf636c3db81afd556d0e95d91b72b9c7 Mon Sep 17 00:00:00 2001
From: Vid <vid.smole@gmail.com>
Date: Thu, 18 Mar 2021 18:53:06 +0100
Subject: [PATCH 309/860] [arnes] Add new extractor(closes #28483)

---
 youtube_dl/extractor/arnes.py      | 101 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |   1 +
 2 files changed, 102 insertions(+)
 create mode 100644 youtube_dl/extractor/arnes.py

diff --git a/youtube_dl/extractor/arnes.py b/youtube_dl/extractor/arnes.py
new file mode 100644
index 000000000..c0032fcab
--- /dev/null
+++ b/youtube_dl/extractor/arnes.py
@@ -0,0 +1,101 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_parse_qs,
+    compat_urllib_parse_urlparse,
+)
+from ..utils import (
+    float_or_none,
+    int_or_none,
+    parse_iso8601,
+    remove_start,
+)
+
+
+class ArnesIE(InfoExtractor):
+    IE_NAME = 'video.arnes.si'
+    IE_DESC = 'Arnes Video'
+    _VALID_URL = r'https?://video\.arnes\.si/(?:[a-z]{2}/)?(?:watch|embed|api/(?:asset|public/video))/(?P<id>[0-9a-zA-Z]{12})'
+    _TESTS = [{
+        'url': 'https://video.arnes.si/watch/a1qrWTOQfVoU?t=10',
+        'md5': '4d0f4d0a03571b33e1efac25fd4a065d',
+        'info_dict': {
+            'id': 'a1qrWTOQfVoU',
+            'ext': 'mp4',
+            'title': 'Linearna neodvisnost, definicija',
+            'description': 'Linearna neodvisnost, definicija',
+            'license': 'PRIVATE',
+            'creator': 'Polona Oblak',
+            'timestamp': 1585063725,
+            'upload_date': '20200324',
+            'channel': 'Polona Oblak',
+            'channel_id': 'q6pc04hw24cj',
+            'channel_url': 'https://video.arnes.si/?channel=q6pc04hw24cj',
+            'duration': 596.75,
+            'view_count': int,
+            'tags': ['linearna_algebra'],
+            'start_time': 10,
+        }
+    }, {
+        'url': 'https://video.arnes.si/api/asset/s1YjnV7hadlC/play.mp4',
+        'only_matching': True,
+    }, {
+        'url': 'https://video.arnes.si/embed/s1YjnV7hadlC',
+        'only_matching': True,
+    }, {
+        'url': 'https://video.arnes.si/en/watch/s1YjnV7hadlC',
+        'only_matching': True,
+    }, {
+        'url': 'https://video.arnes.si/embed/s1YjnV7hadlC?t=123&hideRelated=1',
+        'only_matching': True,
+    }, {
+        'url': 'https://video.arnes.si/api/public/video/s1YjnV7hadlC',
+        'only_matching': True,
+    }]
+    _BASE_URL = 'https://video.arnes.si'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        video = self._download_json(
+            self._BASE_URL + '/api/public/video/' + video_id, video_id)['data']
+        title = video['title']
+
+        formats = []
+        for media in (video.get('media') or []):
+            media_url = media.get('url')
+            if not media_url:
+                continue
+            formats.append({
+                'url': self._BASE_URL + media_url,
+                'format_id': remove_start(media.get('format'), 'FORMAT_'),
+                'format_note': media.get('formatTranslation'),
+                'width': int_or_none(media.get('width')),
+                'height': int_or_none(media.get('height')),
+            })
+        self._sort_formats(formats)
+
+        channel = video.get('channel') or {}
+        channel_id = channel.get('url')
+        thumbnail = video.get('thumbnailUrl')
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnail': self._BASE_URL + thumbnail,
+            'description': video.get('description'),
+            'license': video.get('license'),
+            'creator': video.get('author'),
+            'timestamp': parse_iso8601(video.get('creationTime')),
+            'channel': channel.get('name'),
+            'channel_id': channel_id,
+            'channel_url': self._BASE_URL + '/?channel=' + channel_id if channel_id else None,
+            'duration': float_or_none(video.get('duration'), 1000),
+            'view_count': int_or_none(video.get('views')),
+            'tags': video.get('hashtags'),
+            'start_time': int_or_none(compat_parse_qs(
+                compat_urllib_parse_urlparse(url).query).get('t', [None])[0]),
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b2b39e4dd..8cf348772 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -72,6 +72,7 @@ from .arte import (
     ArteTVEmbedIE,
     ArteTVPlaylistIE,
 )
+from .arnes import ArnesIE
 from .asiancrush import (
     AsianCrushIE,
     AsianCrushPlaylistIE,

From 392c467f95cbf89114235038e1938c72d97144d9 Mon Sep 17 00:00:00 2001
From: Allan Daemon <allan@comp.ufu.br>
Date: Mon, 15 May 2017 00:04:39 -0300
Subject: [PATCH 310/860] [palcomp3] Add new extractor(closes #13120)

---
 youtube_dl/extractor/extractors.py |   5 +
 youtube_dl/extractor/palcomp3.py   | 148 +++++++++++++++++++++++++++++
 2 files changed, 153 insertions(+)
 create mode 100644 youtube_dl/extractor/palcomp3.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 8cf348772..65fefabe8 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -879,6 +879,11 @@ from .packtpub import (
     PacktPubIE,
     PacktPubCourseIE,
 )
+from .palcomp3 import (
+    PalcoMP3IE,
+    PalcoMP3ArtistIE,
+    PalcoMP3VideoIE,
+)
 from .pandoratv import PandoraTVIE
 from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
diff --git a/youtube_dl/extractor/palcomp3.py b/youtube_dl/extractor/palcomp3.py
new file mode 100644
index 000000000..fb29d83f9
--- /dev/null
+++ b/youtube_dl/extractor/palcomp3.py
@@ -0,0 +1,148 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    str_or_none,
+    try_get,
+)
+
+
+class PalcoMP3BaseIE(InfoExtractor):
+    _GQL_QUERY_TMPL = '''{
+  artist(slug: "%s") {
+    %s
+  }
+}'''
+    _ARTIST_FIELDS_TMPL = '''music(slug: "%%s") {
+      %s
+    }'''
+    _MUSIC_FIELDS = '''duration
+      hls
+      mp3File
+      musicID
+      plays
+      title'''
+
+    def _call_api(self, artist_slug, artist_fields):
+        return self._download_json(
+            'https://www.palcomp3.com.br/graphql/', artist_slug, query={
+                'query': self._GQL_QUERY_TMPL % (artist_slug, artist_fields),
+            })['data']
+
+    def _parse_music(self, music):
+        music_id = compat_str(music['musicID'])
+        title = music['title']
+
+        formats = []
+        hls_url = music.get('hls')
+        if hls_url:
+            formats.append({
+                'url': hls_url,
+                'protocol': 'm3u8_native',
+                'ext': 'mp4',
+            })
+        mp3_file = music.get('mp3File')
+        if mp3_file:
+            formats.append({
+                'url': mp3_file,
+            })
+
+        return {
+            'id': music_id,
+            'title': title,
+            'formats': formats,
+            'duration': int_or_none(music.get('duration')),
+            'view_count': int_or_none(music.get('plays')),
+        }
+
+    def _real_initialize(self):
+        self._ARTIST_FIELDS_TMPL = self._ARTIST_FIELDS_TMPL % self._MUSIC_FIELDS
+
+    def _real_extract(self, url):
+        artist_slug, music_slug = re.match(self._VALID_URL, url).groups()
+        artist_fields = self._ARTIST_FIELDS_TMPL % music_slug
+        music = self._call_api(artist_slug, artist_fields)['artist']['music']
+        return self._parse_music(music)
+
+
+class PalcoMP3IE(PalcoMP3BaseIE):
+    IE_NAME = 'PalcoMP3:song'
+    _VALID_URL = r'https?://(?:www\.)?palcomp3\.com(?:\.br)?/(?P<artist>[^/]+)/(?P<id>[^/?&#]+)'
+    _TESTS = [{
+        'url': 'https://www.palcomp3.com/maiaraemaraisaoficial/nossas-composicoes-cuida-bem-dela/',
+        'md5': '99fd6405b2d8fd589670f6db1ba3b358',
+        'info_dict': {
+            'id': '3162927',
+            'ext': 'mp3',
+            'title': 'Nossas Composições - CUIDA BEM DELA',
+            'duration': 210,
+            'view_count': int,
+        }
+    }]
+
+    @classmethod
+    def suitable(cls, url):
+        return False if PalcoMP3VideoIE.suitable(url) else super(PalcoMP3IE, cls).suitable(url)
+
+
+class PalcoMP3ArtistIE(PalcoMP3BaseIE):
+    IE_NAME = 'PalcoMP3:artist'
+    _VALID_URL = r'https?://(?:www\.)?palcomp3\.com(?:\.br)?/(?P<id>[^/?&#]+)'
+    _TESTS = [{
+        'url': 'https://www.palcomp3.com.br/condedoforro/',
+        'info_dict': {
+            'id': '358396',
+            'title': 'Conde do Forró',
+        },
+        'playlist_mincount': 188,
+    }]
+    _ARTIST_FIELDS_TMPL = '''artistID
+    musics {
+      nodes {
+        %s
+      }
+    }
+    name'''
+
+    @ classmethod
+    def suitable(cls, url):
+        return False if re.match(PalcoMP3IE._VALID_URL, url) else super(PalcoMP3ArtistIE, cls).suitable(url)
+
+    def _real_extract(self, url):
+        artist_slug = self._match_id(url)
+        artist = self._call_api(artist_slug, self._ARTIST_FIELDS_TMPL)['artist']
+
+        def entries():
+            for music in (try_get(artist, lambda x: x['musics']['nodes'], list) or []):
+                yield self._parse_music(music)
+
+        return self.playlist_result(
+            entries(), str_or_none(artist.get('artistID')), artist.get('name'))
+
+
+class PalcoMP3VideoIE(PalcoMP3BaseIE):
+    IE_NAME = 'PalcoMP3:video'
+    _VALID_URL = r'https?://(?:www\.)?palcomp3\.com(?:\.br)?/(?P<artist>[^/]+)/(?P<id>[^/?&#]+)/?#clipe'
+    _TESTS = [{
+        'url': 'https://www.palcomp3.com/maiaraemaraisaoficial/maiara-e-maraisa-voce-faz-falta-aqui-ao-vivo-em-vicosa-mg/#clipe',
+        'add_ie': ['Youtube'],
+        'info_dict': {
+            'id': '_pD1nR2qqPg',
+            'ext': 'mp4',
+            'title': 'Maiara e Maraisa - Você Faz Falta Aqui - DVD Ao Vivo Em Campo Grande',
+            'description': 'md5:7043342c09a224598e93546e98e49282',
+            'upload_date': '20161107',
+            'uploader_id': 'maiaramaraisaoficial',
+            'uploader': 'Maiara e Maraisa',
+        }
+    }]
+    _MUSIC_FIELDS = 'youtubeID'
+
+    def _parse_music(self, music):
+        youtube_id = music['youtubeID']
+        return self.url_result(youtube_id, 'Youtube', youtube_id)

From 04d4a3b136060158438c3f2c1b31c884c6961712 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 1 Apr 2021 19:03:45 +0100
Subject: [PATCH 311/860] [screencastomatic] fix extraction(closes #11976,
 closes #24489)

---
 youtube_dl/extractor/screencastomatic.py | 48 +++++++++++++++---------
 1 file changed, 31 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/screencastomatic.py b/youtube_dl/extractor/screencastomatic.py
index b5e76c9af..0afdc1715 100644
--- a/youtube_dl/extractor/screencastomatic.py
+++ b/youtube_dl/extractor/screencastomatic.py
@@ -2,12 +2,18 @@
 from __future__ import unicode_literals
 
 from .common import InfoExtractor
-from ..utils import js_to_json
+from ..utils import (
+    get_element_by_class,
+    int_or_none,
+    remove_start,
+    strip_or_none,
+    unified_strdate,
+)
 
 
 class ScreencastOMaticIE(InfoExtractor):
-    _VALID_URL = r'https?://screencast-o-matic\.com/watch/(?P<id>[0-9a-zA-Z]+)'
-    _TEST = {
+    _VALID_URL = r'https?://screencast-o-matic\.com/(?:(?:watch|player)/|embed\?.*?\bsc=)(?P<id>[0-9a-zA-Z]+)'
+    _TESTS = [{
         'url': 'http://screencast-o-matic.com/watch/c2lD3BeOPl',
         'md5': '483583cb80d92588f15ccbedd90f0c18',
         'info_dict': {
@@ -16,22 +22,30 @@ class ScreencastOMaticIE(InfoExtractor):
             'title': 'Welcome to 3-4 Philosophy @ DECV!',
             'thumbnail': r're:^https?://.*\.jpg$',
             'description': 'as the title says! also: some general info re 1) VCE philosophy and 2) distance learning.',
-            'duration': 369.163,
+            'duration': 369,
+            'upload_date': '20141216',
         }
-    }
+    }, {
+        'url': 'http://screencast-o-matic.com/player/c2lD3BeOPl',
+        'only_matching': True,
+    }, {
+        'url': 'http://screencast-o-matic.com/embed?ff=true&sc=cbV2r4Q5TL&fromPH=true&a=1',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        jwplayer_data = self._parse_json(
-            self._search_regex(
-                r"(?s)jwplayer\('mp4Player'\).setup\((\{.*?\})\);", webpage, 'setup code'),
-            video_id, transform_source=js_to_json)
-
-        info_dict = self._parse_jwplayer_data(jwplayer_data, video_id, require_title=False)
-        info_dict.update({
-            'title': self._og_search_title(webpage),
-            'description': self._og_search_description(webpage),
+        webpage = self._download_webpage(
+            'https://screencast-o-matic.com/player/' + video_id, video_id)
+        info = self._parse_html5_media_entries(url, webpage, video_id)[0]
+        info.update({
+            'id': video_id,
+            'title': get_element_by_class('overlayTitle', webpage),
+            'description': strip_or_none(get_element_by_class('overlayDescription', webpage)) or None,
+            'duration': int_or_none(self._search_regex(
+                r'player\.duration\s*=\s*function\(\)\s*{\s*return\s+(\d+);\s*};',
+                webpage, 'duration', default=None)),
+            'upload_date': unified_strdate(remove_start(
+                get_element_by_class('overlayPublished', webpage), 'Published: ')),
         })
-        return info_dict
+        return info

From 1df2596f81695bf452ffbfd89596d115d9b2daf5 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 3 Apr 2021 07:54:02 +0100
Subject: [PATCH 312/860] [extractor/common] fix _get_cookies method for python
 2(#20673, #23256, #20326, closes #28640)

---
 youtube_dl/extractor/common.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d3b6724df..fcbf18ee6 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2899,7 +2899,10 @@ class InfoExtractor(object):
         """ Return a compat_cookies.SimpleCookie with the cookies for the url """
         req = sanitized_Request(url)
         self._downloader.cookiejar.add_cookie_header(req)
-        return compat_cookies.SimpleCookie(req.get_header('Cookie'))
+        cookie = req.get_header('Cookie')
+        if cookie and sys.version_info[0] == 2:
+            cookie = str(cookie)
+        return compat_cookies.SimpleCookie(cookie)
 
     def _apply_first_set_cookie_header(self, url_handle, cookie):
         """

From 654b4f4ff2718f38b3182c1188c5d569c14cc70a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sat, 3 Apr 2021 08:23:35 +0100
Subject: [PATCH 313/860] [youtube] prioritize information from YoutubeIE for
 playlist entries(closes #28619, closes #28636)

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1f5497e24..2e027528d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -329,7 +329,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             (lambda x: x['ownerText']['runs'][0]['text'],
              lambda x: x['shortBylineText']['runs'][0]['text']), compat_str)
         return {
-            '_type': 'url_transparent',
+            '_type': 'url',
             'ie_key': YoutubeIE.ie_key(),
             'id': video_id,
             'url': video_id,

From aee6feb02adaa316455ea9497e92cc82b720f231 Mon Sep 17 00:00:00 2001
From: RomanEmelyanov <RomanEmelyanov@users.noreply.github.com>
Date: Sun, 4 Apr 2021 11:14:37 +0300
Subject: [PATCH 314/860] [youku] Update ccode(closes #17852, closes #28447,
 closes #28460) (#28648)

---
 youtube_dl/extractor/youku.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youku.py b/youtube_dl/extractor/youku.py
index 61d1ab209..880c89687 100644
--- a/youtube_dl/extractor/youku.py
+++ b/youtube_dl/extractor/youku.py
@@ -154,7 +154,7 @@ class YoukuIE(InfoExtractor):
         # request basic data
         basic_data_params = {
             'vid': video_id,
-            'ccode': '0590',
+            'ccode': '0532',
             'client_ip': '192.168.1.1',
             'utid': cna,
             'client_ts': time.time() / 1000,

From e165f5641fdf62975d3b6a40132a475c9cbaea2a Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 4 Apr 2021 15:52:14 +0100
Subject: [PATCH 315/860] [extractor/common] fix JSON-LD VideoObject author
 extraction

---
 youtube_dl/extractor/common.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index fcbf18ee6..8ef22779a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -70,6 +70,7 @@ from ..utils import (
     str_or_none,
     str_to_int,
     strip_or_none,
+    try_get,
     unescapeHTML,
     unified_strdate,
     unified_timestamp,
@@ -1282,7 +1283,7 @@ class InfoExtractor(object):
                 'thumbnail': url_or_none(e.get('thumbnailUrl') or e.get('thumbnailURL')),
                 'duration': parse_duration(e.get('duration')),
                 'timestamp': unified_timestamp(e.get('uploadDate')),
-                'uploader': str_or_none(e.get('author')),
+                'uploader': try_get(e, lambda x: x['author']['name'], compat_str),
                 'filesize': float_or_none(e.get('contentSize')),
                 'tbr': int_or_none(e.get('bitrate')),
                 'width': int_or_none(e.get('width')),

From 3ae9c0f410b1d4f63e8bada67dd62a8d2852be32 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 4 Apr 2021 16:26:56 +0100
Subject: [PATCH 316/860] [vimeo] improve extraction(closes #28591)

---
 youtube_dl/extractor/vimeo.py | 239 ++++++++++++++--------------------
 1 file changed, 100 insertions(+), 139 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index a90cf0630..102687b82 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 
 import base64
 import functools
-import json
 import re
 import itertools
 
@@ -17,15 +16,14 @@ from ..compat import (
 from ..utils import (
     clean_html,
     determine_ext,
-    dict_get,
     ExtractorError,
+    get_element_by_class,
     js_to_json,
     int_or_none,
     merge_dicts,
     OnDemandPagedList,
     parse_filesize,
     parse_iso8601,
-    RegexNotFoundError,
     sanitized_Request,
     smuggle_url,
     std_headers,
@@ -127,10 +125,11 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         video_title = video_data['title']
         live_event = video_data.get('live_event') or {}
         is_live = live_event.get('status') == 'started'
+        request = config.get('request') or {}
 
         formats = []
-        config_files = video_data.get('files') or config['request'].get('files', {})
-        for f in config_files.get('progressive', []):
+        config_files = video_data.get('files') or request.get('files') or {}
+        for f in (config_files.get('progressive') or []):
             video_url = f.get('url')
             if not video_url:
                 continue
@@ -146,7 +145,7 @@ class VimeoBaseInfoExtractor(InfoExtractor):
         # TODO: fix handling of 308 status code returned for live archive manifest requests
         sep_pattern = r'/sep/video/'
         for files_type in ('hls', 'dash'):
-            for cdn_name, cdn_data in config_files.get(files_type, {}).get('cdns', {}).items():
+            for cdn_name, cdn_data in (try_get(config_files, lambda x: x[files_type]['cdns']) or {}).items():
                 manifest_url = cdn_data.get('url')
                 if not manifest_url:
                     continue
@@ -192,17 +191,15 @@ class VimeoBaseInfoExtractor(InfoExtractor):
                 f['preference'] = -40
 
         subtitles = {}
-        text_tracks = config['request'].get('text_tracks')
-        if text_tracks:
-            for tt in text_tracks:
-                subtitles[tt['lang']] = [{
-                    'ext': 'vtt',
-                    'url': urljoin('https://vimeo.com', tt['url']),
-                }]
+        for tt in (request.get('text_tracks') or []):
+            subtitles[tt['lang']] = [{
+                'ext': 'vtt',
+                'url': urljoin('https://vimeo.com', tt['url']),
+            }]
 
         thumbnails = []
         if not is_live:
-            for key, thumb in video_data.get('thumbs', {}).items():
+            for key, thumb in (video_data.get('thumbs') or {}).items():
                 thumbnails.append({
                     'id': key,
                     'width': int_or_none(key),
@@ -322,6 +319,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'duration': 1595,
                 'upload_date': '20130610',
                 'timestamp': 1370893156,
+                'license': 'by',
             },
             'params': {
                 'format': 'best[protocol=https]',
@@ -400,6 +398,12 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 'uploader_id': 'staff',
                 'uploader': 'Vimeo Staff',
                 'duration': 62,
+                'subtitles': {
+                    'de': [{'ext': 'vtt'}],
+                    'en': [{'ext': 'vtt'}],
+                    'es': [{'ext': 'vtt'}],
+                    'fr': [{'ext': 'vtt'}],
+                },
             }
         },
         {
@@ -572,6 +576,37 @@ class VimeoIE(VimeoBaseInfoExtractor):
     def _real_initialize(self):
         self._login()
 
+    def _extract_from_api(self, video_id, unlisted_hash=None):
+        token = self._download_json(
+            'https://vimeo.com/_rv/jwt', video_id, headers={
+                'X-Requested-With': 'XMLHttpRequest'
+            })['token']
+        api_url = 'https://api.vimeo.com/videos/' + video_id
+        if unlisted_hash:
+            api_url += ':' + unlisted_hash
+        video = self._download_json(
+            api_url, video_id, headers={
+                'Authorization': 'jwt ' + token,
+            }, query={
+                'fields': 'config_url,created_time,description,license,metadata.connections.comments.total,metadata.connections.likes.total,release_time,stats.plays',
+            })
+        info = self._parse_config(self._download_json(
+            video['config_url'], video_id), video_id)
+        self._vimeo_sort_formats(info['formats'])
+        get_timestamp = lambda x: parse_iso8601(video.get(x + '_time'))
+        info.update({
+            'description': video.get('description'),
+            'license': video.get('license'),
+            'release_timestamp': get_timestamp('release'),
+            'timestamp': get_timestamp('created'),
+            'view_count': int_or_none(try_get(video, lambda x: x['stats']['plays'])),
+        })
+        connections = try_get(
+            video, lambda x: x['metadata']['connections'], dict) or {}
+        for k in ('comment', 'like'):
+            info[k + '_count'] = int_or_none(try_get(connections, lambda x: x[k + 's']['total']))
+        return info
+
     def _real_extract(self, url):
         url, data = unsmuggle_url(url, {})
         headers = std_headers.copy()
@@ -580,48 +615,19 @@ class VimeoIE(VimeoBaseInfoExtractor):
         if 'Referer' not in headers:
             headers['Referer'] = url
 
-        # Extract ID from URL
-        video_id, unlisted_hash = re.match(self._VALID_URL, url).groups()
+        mobj = re.match(self._VALID_URL, url).groupdict()
+        video_id, unlisted_hash = mobj['id'], mobj.get('unlisted_hash')
         if unlisted_hash:
-            token = self._download_json(
-                'https://vimeo.com/_rv/jwt', video_id, headers={
-                    'X-Requested-With': 'XMLHttpRequest'
-                })['token']
-            video = self._download_json(
-                'https://api.vimeo.com/videos/%s:%s' % (video_id, unlisted_hash),
-                video_id, headers={
-                    'Authorization': 'jwt ' + token,
-                }, query={
-                    'fields': 'config_url,created_time,description,license,metadata.connections.comments.total,metadata.connections.likes.total,release_time,stats.plays',
-                })
-            info = self._parse_config(self._download_json(
-                video['config_url'], video_id), video_id)
-            self._vimeo_sort_formats(info['formats'])
-            get_timestamp = lambda x: parse_iso8601(video.get(x + '_time'))
-            info.update({
-                'description': video.get('description'),
-                'license': video.get('license'),
-                'release_timestamp': get_timestamp('release'),
-                'timestamp': get_timestamp('created'),
-                'view_count': int_or_none(try_get(video, lambda x: x['stats']['plays'])),
-            })
-            connections = try_get(
-                video, lambda x: x['metadata']['connections'], dict) or {}
-            for k in ('comment', 'like'):
-                info[k + '_count'] = int_or_none(try_get(connections, lambda x: x[k + 's']['total']))
-            return info
+            return self._extract_from_api(video_id, unlisted_hash)
 
         orig_url = url
         is_pro = 'vimeopro.com/' in url
-        is_player = '://player.vimeo.com/video/' in url
         if is_pro:
             # some videos require portfolio_id to be present in player url
             # https://github.com/ytdl-org/youtube-dl/issues/20070
             url = self._extract_url(url, self._download_webpage(url, video_id))
             if not url:
                 url = 'https://vimeo.com/' + video_id
-        elif is_player:
-            url = 'https://player.vimeo.com/video/' + video_id
         elif any(p in url for p in ('play_redirect_hls', 'moogaloop.swf')):
             url = 'https://vimeo.com/' + video_id
 
@@ -641,14 +647,25 @@ class VimeoIE(VimeoBaseInfoExtractor):
                         expected=True)
             raise
 
-        # Now we begin extracting as much information as we can from what we
-        # retrieved. First we extract the information common to all extractors,
-        # and latter we extract those that are Vimeo specific.
-        self.report_extraction(video_id)
+        if '://player.vimeo.com/video/' in url:
+            config = self._parse_json(self._search_regex(
+                r'\bconfig\s*=\s*({.+?})\s*;', webpage, 'info section'), video_id)
+            if config.get('view') == 4:
+                config = self._verify_player_video_password(
+                    redirect_url, video_id, headers)
+            info = self._parse_config(config, video_id)
+            self._vimeo_sort_formats(info['formats'])
+            return info
+
+        if re.search(r'<form[^>]+?id="pw_form"', webpage):
+            video_password = self._get_video_password()
+            token, vuid = self._extract_xsrft_and_vuid(webpage)
+            webpage = self._verify_video_password(
+                redirect_url, video_id, video_password, token, vuid)
 
         vimeo_config = self._extract_vimeo_config(webpage, video_id, default=None)
         if vimeo_config:
-            seed_status = vimeo_config.get('seed_status', {})
+            seed_status = vimeo_config.get('seed_status') or {}
             if seed_status.get('state') == 'failed':
                 raise ExtractorError(
                     '%s said: %s' % (self.IE_NAME, seed_status['title']),
@@ -657,70 +674,40 @@ class VimeoIE(VimeoBaseInfoExtractor):
         cc_license = None
         timestamp = None
         video_description = None
+        info_dict = {}
 
-        # Extract the config JSON
-        try:
-            try:
-                config_url = self._html_search_regex(
-                    r' data-config-url="(.+?)"', webpage,
-                    'config URL', default=None)
-                if not config_url:
-                    # Sometimes new react-based page is served instead of old one that require
-                    # different config URL extraction approach (see
-                    # https://github.com/ytdl-org/youtube-dl/pull/7209)
-                    page_config = self._parse_json(self._search_regex(
-                        r'vimeo\.(?:clip|vod_title)_page_config\s*=\s*({.+?});',
-                        webpage, 'page config'), video_id)
-                    config_url = page_config['player']['config_url']
-                    cc_license = page_config.get('cc_license')
-                    timestamp = try_get(
-                        page_config, lambda x: x['clip']['uploaded_on'],
-                        compat_str)
-                    video_description = clean_html(dict_get(
-                        page_config, ('description', 'description_html_escaped')))
-                config = self._download_json(config_url, video_id)
-            except RegexNotFoundError:
-                # For pro videos or player.vimeo.com urls
-                # We try to find out to which variable is assigned the config dic
-                m_variable_name = re.search(r'(\w)\.video\.id', webpage)
-                if m_variable_name is not None:
-                    config_re = [r'%s=({[^}].+?});' % re.escape(m_variable_name.group(1))]
-                else:
-                    config_re = [r' = {config:({.+?}),assets:', r'(?:[abc])=({.+?});']
-                config_re.append(r'\bvar\s+r\s*=\s*({.+?})\s*;')
-                config_re.append(r'\bconfig\s*=\s*({.+?})\s*;')
-                config = self._search_regex(config_re, webpage, 'info section',
-                                            flags=re.DOTALL)
-                config = json.loads(config)
-        except Exception as e:
-            if re.search('The creator of this video has not given you permission to embed it on this domain.', webpage):
-                raise ExtractorError('The author has restricted the access to this video, try with the "--referer" option')
-
-            if re.search(r'<form[^>]+?id="pw_form"', webpage) is not None:
-                if '_video_password_verified' in data:
-                    raise ExtractorError('video password verification failed!')
-                video_password = self._get_video_password()
-                token, vuid = self._extract_xsrft_and_vuid(webpage)
-                self._verify_video_password(
-                    redirect_url, video_id, video_password, token, vuid)
-                return self._real_extract(
-                    smuggle_url(redirect_url, {'_video_password_verified': 'verified'}))
-            else:
-                raise ExtractorError('Unable to extract info section',
-                                     cause=e)
+        channel_id = self._search_regex(
+            r'vimeo\.com/channels/([^/]+)', url, 'channel id', default=None)
+        if channel_id:
+            config_url = self._html_search_regex(
+                r'\bdata-config-url="([^"]+)"', webpage, 'config URL')
+            video_description = clean_html(get_element_by_class('description', webpage))
+            info_dict.update({
+                'channel_id': channel_id,
+                'channel_url': 'https://vimeo.com/channels/' + channel_id,
+            })
         else:
-            if config.get('view') == 4:
-                config = self._verify_player_video_password(redirect_url, video_id, headers)
-
+            page_config = self._parse_json(self._search_regex(
+                r'vimeo\.(?:clip|vod_title)_page_config\s*=\s*({.+?});',
+                webpage, 'page config', default='{}'), video_id, fatal=False)
+            if not page_config:
+                return self._extract_from_api(video_id)
+            config_url = page_config['player']['config_url']
+            cc_license = page_config.get('cc_license')
+            clip = page_config.get('clip') or {}
+            timestamp = clip.get('uploaded_on')
+            video_description = clean_html(
+                clip.get('description') or page_config.get('description_html_escaped'))
+        config = self._download_json(config_url, video_id)
         video = config.get('video') or {}
         vod = video.get('vod') or {}
 
         def is_rented():
             if '>You rented this title.<' in webpage:
                 return True
-            if config.get('user', {}).get('purchased'):
+            if try_get(config, lambda x: x['user']['purchased']):
                 return True
-            for purchase_option in vod.get('purchase_options', []):
+            for purchase_option in (vod.get('purchase_options') or []):
                 if purchase_option.get('purchased'):
                     return True
                 label = purchase_option.get('label_string')
@@ -735,14 +722,10 @@ class VimeoIE(VimeoBaseInfoExtractor):
                     'https://player.vimeo.com/player/%s' % feature_id,
                     {'force_feature_id': True}), 'Vimeo')
 
-        # Extract video description
-        if not video_description:
-            video_description = self._html_search_regex(
-                r'(?s)<div\s+class="[^"]*description[^"]*"[^>]*>(.*?)</div>',
-                webpage, 'description', default=None)
         if not video_description:
             video_description = self._html_search_meta(
-                'description', webpage, default=None)
+                ['description', 'og:description', 'twitter:description'],
+                webpage, default=None)
         if not video_description and is_pro:
             orig_webpage = self._download_webpage(
                 orig_url, video_id,
@@ -751,25 +734,14 @@ class VimeoIE(VimeoBaseInfoExtractor):
             if orig_webpage:
                 video_description = self._html_search_meta(
                     'description', orig_webpage, default=None)
-        if not video_description and not is_player:
+        if not video_description:
             self._downloader.report_warning('Cannot find video description')
 
-        # Extract upload date
         if not timestamp:
             timestamp = self._search_regex(
                 r'<time[^>]+datetime="([^"]+)"', webpage,
                 'timestamp', default=None)
 
-        try:
-            view_count = int(self._search_regex(r'UserPlays:(\d+)', webpage, 'view count'))
-            like_count = int(self._search_regex(r'UserLikes:(\d+)', webpage, 'like count'))
-            comment_count = int(self._search_regex(r'UserComments:(\d+)', webpage, 'comment count'))
-        except RegexNotFoundError:
-            # This info is only available in vimeo.com/{id} urls
-            view_count = None
-            like_count = None
-            comment_count = None
-
         formats = []
 
         source_format = self._extract_original_format(
@@ -788,31 +760,20 @@ class VimeoIE(VimeoBaseInfoExtractor):
                 r'<link[^>]+rel=["\']license["\'][^>]+href=(["\'])(?P<license>(?:(?!\1).)+)\1',
                 webpage, 'license', default=None, group='license')
 
-        channel_id = self._search_regex(
-            r'vimeo\.com/channels/([^/]+)', url, 'channel id', default=None)
-        channel_url = 'https://vimeo.com/channels/%s' % channel_id if channel_id else None
-
-        info_dict = {
+        info_dict.update({
             'formats': formats,
             'timestamp': unified_timestamp(timestamp),
             'description': video_description,
             'webpage_url': url,
-            'view_count': view_count,
-            'like_count': like_count,
-            'comment_count': comment_count,
             'license': cc_license,
-            'channel_id': channel_id,
-            'channel_url': channel_url,
-        }
+        })
 
-        info_dict = merge_dicts(info_dict, info_dict_config, json_ld)
-
-        return info_dict
+        return merge_dicts(info_dict, info_dict_config, json_ld)
 
 
 class VimeoOndemandIE(VimeoIE):
     IE_NAME = 'vimeo:ondemand'
-    _VALID_URL = r'https?://(?:www\.)?vimeo\.com/ondemand/([^/]+/)?(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?vimeo\.com/ondemand/(?:[^/]+/)?(?P<id>[^/?#&]+)'
     _TESTS = [{
         # ondemand video not available via https://vimeo.com/id
         'url': 'https://vimeo.com/ondemand/20704',

From 6beb1ac65b03415764c487fd139298f22e1e0313 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 4 Apr 2021 19:16:17 +0100
Subject: [PATCH 317/860] [extractor/common] keep support for non standard
 JSON-LD VideoObject author values

---
 youtube_dl/extractor/common.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 8ef22779a..78ff5b6d0 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -70,7 +70,6 @@ from ..utils import (
     str_or_none,
     str_to_int,
     strip_or_none,
-    try_get,
     unescapeHTML,
     unified_strdate,
     unified_timestamp,
@@ -1276,6 +1275,7 @@ class InfoExtractor(object):
 
         def extract_video_object(e):
             assert e['@type'] == 'VideoObject'
+            author = e.get('author')
             info.update({
                 'url': url_or_none(e.get('contentUrl')),
                 'title': unescapeHTML(e.get('name')),
@@ -1283,7 +1283,11 @@ class InfoExtractor(object):
                 'thumbnail': url_or_none(e.get('thumbnailUrl') or e.get('thumbnailURL')),
                 'duration': parse_duration(e.get('duration')),
                 'timestamp': unified_timestamp(e.get('uploadDate')),
-                'uploader': try_get(e, lambda x: x['author']['name'], compat_str),
+                # author can be an instance of 'Organization' or 'Person' types.
+                # both types can have 'name' property(inherited from 'Thing' type). [1]
+                # however some websites are using 'Text' type instead.
+                # 1. https://schema.org/VideoObject
+                'uploader': author.get('name') if isinstance(author, dict) else author if isinstance(author, compat_str) else None,
                 'filesize': float_or_none(e.get('contentSize')),
                 'tbr': int_or_none(e.get('bitrate')),
                 'width': int_or_none(e.get('width')),

From 162bf9e10a4e6a08f5ed156a68054ef9b4d2b60e Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Sun, 4 Apr 2021 19:49:24 +0100
Subject: [PATCH 318/860] [compat] add compat_SimpleCookie

---
 youtube_dl/compat.py           | 9 +++++++++
 youtube_dl/extractor/common.py | 9 +++------
 2 files changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 6c3d49d45..8bbebebcf 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -73,6 +73,15 @@ try:
 except ImportError:  # Python 2
     import Cookie as compat_cookies
 
+if sys.version_info[0] == 2:
+    class compat_SimpleCookie(compat_cookies.SimpleCookie):
+        def load(self, rawdata):
+            if isinstance(rawdata, unicode):
+                rawdata = str(rawdata)
+            return super(compat_SimpleCookie, self).load(rawdata)
+else:
+    compat_SimpleCookie = compat_cookies.SimpleCookie
+
 try:
     import html.entities as compat_html_entities
 except ImportError:  # Python 2
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 78ff5b6d0..af289d705 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -17,13 +17,13 @@ import math
 
 from ..compat import (
     compat_cookiejar_Cookie,
-    compat_cookies,
     compat_etree_Element,
     compat_etree_fromstring,
     compat_getpass,
     compat_integer_types,
     compat_http_client,
     compat_os_name,
+    compat_SimpleCookie,
     compat_str,
     compat_urllib_error,
     compat_urllib_parse_unquote,
@@ -2901,13 +2901,10 @@ class InfoExtractor(object):
         self._downloader.cookiejar.set_cookie(cookie)
 
     def _get_cookies(self, url):
-        """ Return a compat_cookies.SimpleCookie with the cookies for the url """
+        """ Return a compat_SimpleCookie with the cookies for the url """
         req = sanitized_Request(url)
         self._downloader.cookiejar.add_cookie_header(req)
-        cookie = req.get_header('Cookie')
-        if cookie and sys.version_info[0] == 2:
-            cookie = str(cookie)
-        return compat_cookies.SimpleCookie(cookie)
+        return compat_SimpleCookie(req.get_header('Cookie'))
 
     def _apply_first_set_cookie_header(self, url_handle, cookie):
         """

From 760c911299aa607ca967d6d4be2985528bacf29f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 5 Apr 2021 07:16:50 +0100
Subject: [PATCH 319/860] [compat] add compat_SimpleCookie to __all__ array

---
 youtube_dl/compat.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 8bbebebcf..8a5262dc8 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -3002,6 +3002,7 @@ __all__ = [
     'compat_HTMLParseError',
     'compat_HTMLParser',
     'compat_HTTPError',
+    'compat_SimpleCookie',
     'compat_Struct',
     'compat_b64decode',
     'compat_basestring',

From 25b1287323f5836c9416a8183096adc63809d5ce Mon Sep 17 00:00:00 2001
From: guredora <contact@guredora.com>
Date: Sun, 4 Apr 2021 22:12:07 +0900
Subject: [PATCH 320/860] [line] add support live.line.me (closes
 #17205)(closes #28658)

---
 youtube_dl/extractor/extractors.py |   6 +-
 youtube_dl/extractor/line.py       | 142 ++++++++++++++++++++++++++++-
 2 files changed, 146 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 65fefabe8..d5cd364e8 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -595,7 +595,11 @@ from .limelight import (
     LimelightChannelIE,
     LimelightChannelListIE,
 )
-from .line import LineTVIE
+from .line import (
+    LineTVIE,
+    LineLiveIE,
+    LineLiveChannelIE,
+)
 from .linkedin import (
     LinkedInLearningIE,
     LinkedInLearningCourseIE,
diff --git a/youtube_dl/extractor/line.py b/youtube_dl/extractor/line.py
index 7f5fa446e..2526daa77 100644
--- a/youtube_dl/extractor/line.py
+++ b/youtube_dl/extractor/line.py
@@ -4,7 +4,13 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..utils import js_to_json
+from ..compat import compat_str
+from ..utils import (
+    ExtractorError,
+    int_or_none,
+    js_to_json,
+    str_or_none,
+)
 
 
 class LineTVIE(InfoExtractor):
@@ -88,3 +94,137 @@ class LineTVIE(InfoExtractor):
                            for thumbnail in video_info.get('thumbnails', {}).get('list', [])],
             'view_count': video_info.get('meta', {}).get('count'),
         }
+
+
+class LineLiveBaseIE(InfoExtractor):
+    _API_BASE_URL = 'https://live-api.line-apps.com/web/v4.0/channel/'
+
+    def _parse_broadcast_item(self, item):
+        broadcast_id = compat_str(item['id'])
+        title = item['title']
+        is_live = item.get('isBroadcastingNow')
+
+        thumbnails = []
+        for thumbnail_id, thumbnail_url in (item.get('thumbnailURLs') or {}).items():
+            if not thumbnail_url:
+                continue
+            thumbnails.append({
+                'id': thumbnail_id,
+                'url': thumbnail_url,
+            })
+
+        channel = item.get('channel') or {}
+        channel_id = str_or_none(channel.get('id'))
+
+        return {
+            'id': broadcast_id,
+            'title': self._live_title(title) if is_live else title,
+            'thumbnails': thumbnails,
+            'timestamp': int_or_none(item.get('createdAt')),
+            'channel': channel.get('name'),
+            'channel_id': channel_id,
+            'channel_url': 'https://live.line.me/channels/' + channel_id if channel_id else None,
+            'duration': int_or_none(item.get('archiveDuration')),
+            'view_count': int_or_none(item.get('viewerCount')),
+            'comment_count': int_or_none(item.get('chatCount')),
+            'is_live': is_live,
+        }
+
+
+class LineLiveIE(LineLiveBaseIE):
+    _VALID_URL = r'https?://live\.line\.me/channels/(?P<channel_id>\d+)/broadcast/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://live.line.me/channels/4867368/broadcast/16331360',
+        'md5': 'bc931f26bf1d4f971e3b0982b3fab4a3',
+        'info_dict': {
+            'id': '16331360',
+            'title': '振りコピ講座😙😙😙',
+            'ext': 'mp4',
+            'timestamp': 1617095132,
+            'upload_date': '20210330',
+            'channel': '白川ゆめか',
+            'channel_id': '4867368',
+            'view_count': int,
+            'comment_count': int,
+            'is_live': False,
+        }
+    }, {
+        # archiveStatus == 'DELETED'
+        'url': 'https://live.line.me/channels/4778159/broadcast/16378488',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        channel_id, broadcast_id = re.match(self._VALID_URL, url).groups()
+        broadcast = self._download_json(
+            self._API_BASE_URL + '%s/broadcast/%s' % (channel_id, broadcast_id),
+            broadcast_id)
+        item = broadcast['item']
+        info = self._parse_broadcast_item(item)
+        protocol = 'm3u8' if info['is_live'] else 'm3u8_native'
+        formats = []
+        for k, v in (broadcast.get(('live' if info['is_live'] else 'archived') + 'HLSURLs') or {}).items():
+            if not v:
+                continue
+            if k == 'abr':
+                formats.extend(self._extract_m3u8_formats(
+                    v, broadcast_id, 'mp4', protocol,
+                    m3u8_id='hls', fatal=False))
+                continue
+            f = {
+                'ext': 'mp4',
+                'format_id': 'hls-' + k,
+                'protocol': protocol,
+                'url': v,
+            }
+            if not k.isdigit():
+                f['vcodec'] = 'none'
+            formats.append(f)
+        if not formats:
+            archive_status = item.get('archiveStatus')
+            if archive_status != 'ARCHIVED':
+                raise ExtractorError('this video has been ' + archive_status.lower(), expected=True)
+        self._sort_formats(formats)
+        info['formats'] = formats
+        return info
+
+
+class LineLiveChannelIE(LineLiveBaseIE):
+    _VALID_URL = r'https?://live\.line\.me/channels/(?P<id>\d+)(?!/broadcast/\d+)(?:[/?&#]|$)'
+    _TEST = {
+        'url': 'https://live.line.me/channels/5893542',
+        'info_dict': {
+            'id': '5893542',
+            'title': 'いくらちゃん',
+            'description': 'md5:c3a4af801f43b2fac0b02294976580be',
+        },
+        'playlist_mincount': 29
+    }
+
+    def _archived_broadcasts_entries(self, archived_broadcasts, channel_id):
+        while True:
+            for row in (archived_broadcasts.get('rows') or []):
+                share_url = str_or_none(row.get('shareURL'))
+                if not share_url:
+                    continue
+                info = self._parse_broadcast_item(row)
+                info.update({
+                    '_type': 'url',
+                    'url': share_url,
+                    'ie_key': LineLiveIE.ie_key(),
+                })
+                yield info
+            if not archived_broadcasts.get('hasNextPage'):
+                return
+            archived_broadcasts = self._download_json(
+                self._API_BASE_URL + channel_id + '/archived_broadcasts',
+                channel_id, query={
+                    'lastId': info['id'],
+                })
+
+    def _real_extract(self, url):
+        channel_id = self._match_id(url)
+        channel = self._download_json(self._API_BASE_URL + channel_id, channel_id)
+        return self.playlist_result(
+            self._archived_broadcasts_entries(channel.get('archivedBroadcasts') or {}, channel_id),
+            channel_id, channel.get('title'), channel.get('information'))

From 6b315d96bc0b07ddc3abaa7318583775828cce30 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Apr 2021 14:15:13 +0700
Subject: [PATCH 321/860] [compat] flake8

---
 youtube_dl/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 8a5262dc8..566e9d5ec 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -76,7 +76,7 @@ except ImportError:  # Python 2
 if sys.version_info[0] == 2:
     class compat_SimpleCookie(compat_cookies.SimpleCookie):
         def load(self, rawdata):
-            if isinstance(rawdata, unicode):
+            if isinstance(rawdata, compat_str):
                 rawdata = str(rawdata)
             return super(compat_SimpleCookie, self).load(rawdata)
 else:

From 70d0d4f9beba0e5b6d95ee50ad62ae7ab5be9be1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Tue, 6 Apr 2021 14:22:28 +0700
Subject: [PATCH 322/860] [compat] Use more conventional name for compat
 SimpleCookie

---
 youtube_dl/compat.py           | 8 ++++----
 youtube_dl/extractor/common.py | 6 +++---
 2 files changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 566e9d5ec..9e45c454b 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -74,13 +74,13 @@ except ImportError:  # Python 2
     import Cookie as compat_cookies
 
 if sys.version_info[0] == 2:
-    class compat_SimpleCookie(compat_cookies.SimpleCookie):
+    class compat_cookies_SimpleCookie(compat_cookies.SimpleCookie):
         def load(self, rawdata):
             if isinstance(rawdata, compat_str):
                 rawdata = str(rawdata)
-            return super(compat_SimpleCookie, self).load(rawdata)
+            return super(compat_cookies_SimpleCookie, self).load(rawdata)
 else:
-    compat_SimpleCookie = compat_cookies.SimpleCookie
+    compat_cookies_SimpleCookie = compat_cookies.SimpleCookie
 
 try:
     import html.entities as compat_html_entities
@@ -3002,7 +3002,6 @@ __all__ = [
     'compat_HTMLParseError',
     'compat_HTMLParser',
     'compat_HTTPError',
-    'compat_SimpleCookie',
     'compat_Struct',
     'compat_b64decode',
     'compat_basestring',
@@ -3010,6 +3009,7 @@ __all__ = [
     'compat_cookiejar',
     'compat_cookiejar_Cookie',
     'compat_cookies',
+    'compat_cookies_SimpleCookie',
     'compat_ctypes_WINFUNCTYPE',
     'compat_etree_Element',
     'compat_etree_fromstring',
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index af289d705..797c35fd5 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -17,13 +17,13 @@ import math
 
 from ..compat import (
     compat_cookiejar_Cookie,
+    compat_cookies_SimpleCookie,
     compat_etree_Element,
     compat_etree_fromstring,
     compat_getpass,
     compat_integer_types,
     compat_http_client,
     compat_os_name,
-    compat_SimpleCookie,
     compat_str,
     compat_urllib_error,
     compat_urllib_parse_unquote,
@@ -2901,10 +2901,10 @@ class InfoExtractor(object):
         self._downloader.cookiejar.set_cookie(cookie)
 
     def _get_cookies(self, url):
-        """ Return a compat_SimpleCookie with the cookies for the url """
+        """ Return a compat_cookies_SimpleCookie with the cookies for the url """
         req = sanitized_Request(url)
         self._downloader.cookiejar.add_cookie_header(req)
-        return compat_SimpleCookie(req.get_header('Cookie'))
+        return compat_cookies_SimpleCookie(req.get_header('Cookie'))
 
     def _apply_first_set_cookie_header(self, url_handle, cookie):
         """

From 6b116f0c03ac0b1aff01cd08bbe1d5cb87dff853 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Apr 2021 03:34:43 +0700
Subject: [PATCH 323/860] [youtube] Fix videos with restricted location (closes
 #28685)

---
 youtube_dl/extractor/youtube.py | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2e027528d..6b4c7912c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1084,6 +1084,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'url': 'https://www.youtube.com/watch?v=nGC3D_FkCmg',
             'only_matching': True,
         },
+        {
+            # restricted location, https://github.com/ytdl-org/youtube-dl/issues/28685
+            'url': 'cBvYw8_A0vQ',
+            'info_dict': {
+                'id': 'cBvYw8_A0vQ',
+                'ext': 'mp4',
+                'title': '4K Ueno Okachimachi  Street  Scenes  上野御徒町歩き',
+                'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
+                'upload_date': '20201120',
+                'uploader': 'Walk around Japan',
+                'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
+                'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
     ]
     _formats = {
         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},
@@ -1485,7 +1502,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         def get_text(x):
             if not x:
                 return
-            return x.get('simpleText') or ''.join([r['text'] for r in x['runs']])
+            text = x.get('simpleText')
+            if text and isinstance(text, compat_str):
+                return text
+            runs = x.get('runs')
+            if not isinstance(runs, list):
+                return
+            return ''.join([r['text'] for r in runs if isinstance(r.get('text'), compat_str)])
 
         search_meta = (
             lambda x: self._html_search_meta(x, webpage, default=None)) \

From 445db582a27c44cb02d57ac9171d58651cafbd76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Apr 2021 03:35:25 +0700
Subject: [PATCH 324/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 4304ecd9e..e5e546744 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,25 @@
+version <unreleased>
+
+Core
+* [extractor/common] Use compat_cookies_SimpleCookie for _get_cookies
++ [compat] Introduce compat_cookies_SimpleCookie
+* [extractor/common] Improve JSON-LD author extraction
+* [extractor/common] Fix _get_cookies on python 2 (#20673, #23256, #20326,
+  #28640)
+
+Extractors
+* [youtube] Fix extraction of videos with restricted location (#28685)
++ [line] Add support for live.line.me (#17205, #28658)
+* [vimeo] Improve extraction (#28591)
+* [youku] Update ccode (#17852, #28447, #28460, #28648)
+* [youtube] Prefer direct entry metadata over entry metadata from playlist
+  (#28619, #28636)
+* [screencastomatic] Fix extraction (#11976, #24489)
++ [palcomp3] Add support for palcomp3.com (#13120)
++ [arnes] Add support for video.arnes.si (#28483)
++ [youtube:tab] Add support for hashtags (#28308)
+
+
 version 2021.04.01
 
 Extractors

From 72a2c0a9ede04c6b82235e453b1a933faf072a76 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 7 Apr 2021 03:42:24 +0700
Subject: [PATCH 325/860] release 2021.04.07

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 docs/supportedsites.md                           | 6 ++++++
 youtube_dl/version.py                            | 2 +-
 8 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 98ec799e8..febbd2344 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.01. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.07. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.04.01**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.07**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.04.01
+ [debug] youtube-dl version 2021.04.07
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index 5387a6cd1..d7296d0a9 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.01. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.07. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.04.01**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.07**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 945c80366..92e616a1a 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.01. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.07. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.04.01**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.07**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index 0acc8b679..b55739f6c 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.01. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.07. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.04.01**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.07**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.04.01
+ [debug] youtube-dl version 2021.04.07
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index 42c3126a3..dbdb8356a 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.01. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.07. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.04.01**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.07**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index e5e546744..22b4fa67d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.04.07
 
 Core
 * [extractor/common] Use compat_cookies_SimpleCookie for _get_cookies
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index d2ad937a4..ff9177a2c 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -463,6 +463,8 @@
  - **limelight**
  - **limelight:channel**
  - **limelight:channel_list**
+ - **LineLive**
+ - **LineLiveChannel**
  - **LineTV**
  - **linkedin:learning**
  - **linkedin:learning:course**
@@ -679,6 +681,9 @@
  - **OutsideTV**
  - **PacktPub**
  - **PacktPubCourse**
+ - **PalcoMP3:artist**
+ - **PalcoMP3:song**
+ - **PalcoMP3:video**
  - **pandora.tv**: 판도라TV
  - **ParamountNetwork**
  - **parliamentlive.tv**: UK parliament videos
@@ -1059,6 +1064,7 @@
  - **Vidbit**
  - **Viddler**
  - **Videa**
+ - **video.arnes.si**: Arnes Video
  - **video.google:search**: Google Video search
  - **video.sky.it**
  - **video.sky.it:live**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 0457d1a15..a6b1b8dce 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.04.01'
+__version__ = '2021.04.07'

From c0c5134c5771dd2a1caeeaee62dcd207d169e981 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 7 Apr 2021 09:27:05 +0100
Subject: [PATCH 326/860] [curiositystream] fix format extraction(closes
 #26845, closes #28668)

---
 youtube_dl/extractor/curiositystream.py | 103 +++++++++++++-----------
 1 file changed, 58 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/curiositystream.py b/youtube_dl/extractor/curiositystream.py
index e4a7fca6c..ae64a07d7 100644
--- a/youtube_dl/extractor/curiositystream.py
+++ b/youtube_dl/extractor/curiositystream.py
@@ -25,12 +25,12 @@ class CuriosityStreamBaseIE(InfoExtractor):
             raise ExtractorError(
                 '%s said: %s' % (self.IE_NAME, error), expected=True)
 
-    def _call_api(self, path, video_id):
+    def _call_api(self, path, video_id, query=None):
         headers = {}
         if self._auth_token:
             headers['X-Auth-Token'] = self._auth_token
         result = self._download_json(
-            self._API_BASE_URL + path, video_id, headers=headers)
+            self._API_BASE_URL + path, video_id, headers=headers, query=query)
         self._handle_errors(result)
         return result['data']
 
@@ -52,62 +52,75 @@ class CuriosityStreamIE(CuriosityStreamBaseIE):
     _VALID_URL = r'https?://(?:app\.)?curiositystream\.com/video/(?P<id>\d+)'
     _TEST = {
         'url': 'https://app.curiositystream.com/video/2',
-        'md5': '262bb2f257ff301115f1973540de8983',
         'info_dict': {
             'id': '2',
             'ext': 'mp4',
             'title': 'How Did You Develop The Internet?',
             'description': 'Vint Cerf, Google\'s Chief Internet Evangelist, describes how he and Bob Kahn created the internet.',
-        }
+        },
+        'params': {
+            'format': 'bestvideo',
+            # m3u8 download
+            'skip_download': True,
+        },
     }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        media = self._call_api('media/' + video_id, video_id)
-        title = media['title']
 
         formats = []
-        for encoding in media.get('encodings', []):
-            m3u8_url = encoding.get('master_playlist_url')
-            if m3u8_url:
-                formats.extend(self._extract_m3u8_formats(
-                    m3u8_url, video_id, 'mp4', 'm3u8_native',
-                    m3u8_id='hls', fatal=False))
-            encoding_url = encoding.get('url')
-            file_url = encoding.get('file_url')
-            if not encoding_url and not file_url:
-                continue
-            f = {
-                'width': int_or_none(encoding.get('width')),
-                'height': int_or_none(encoding.get('height')),
-                'vbr': int_or_none(encoding.get('video_bitrate')),
-                'abr': int_or_none(encoding.get('audio_bitrate')),
-                'filesize': int_or_none(encoding.get('size_in_bytes')),
-                'vcodec': encoding.get('video_codec'),
-                'acodec': encoding.get('audio_codec'),
-                'container': encoding.get('container_type'),
-            }
-            for f_url in (encoding_url, file_url):
-                if not f_url:
+        for encoding_format in ('m3u8', 'mpd'):
+            media = self._call_api('media/' + video_id, video_id, query={
+                'encodingsNew': 'true',
+                'encodingsFormat': encoding_format,
+            })
+            for encoding in media.get('encodings', []):
+                playlist_url = encoding.get('master_playlist_url')
+                if encoding_format == 'm3u8':
+                    # use `m3u8` entry_protocol until EXT-X-MAP is properly supported by `m3u8_native` entry_protocol
+                    formats.extend(self._extract_m3u8_formats(
+                        playlist_url, video_id, 'mp4',
+                        m3u8_id='hls', fatal=False))
+                elif encoding_format == 'mpd':
+                    formats.extend(self._extract_mpd_formats(
+                        playlist_url, video_id, mpd_id='dash', fatal=False))
+                encoding_url = encoding.get('url')
+                file_url = encoding.get('file_url')
+                if not encoding_url and not file_url:
                     continue
-                fmt = f.copy()
-                rtmp = re.search(r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+))/(?P<playpath>mp[34]:.+)$', f_url)
-                if rtmp:
-                    fmt.update({
-                        'url': rtmp.group('url'),
-                        'play_path': rtmp.group('playpath'),
-                        'app': rtmp.group('app'),
-                        'ext': 'flv',
-                        'format_id': 'rtmp',
-                    })
-                else:
-                    fmt.update({
-                        'url': f_url,
-                        'format_id': 'http',
-                    })
-                formats.append(fmt)
+                f = {
+                    'width': int_or_none(encoding.get('width')),
+                    'height': int_or_none(encoding.get('height')),
+                    'vbr': int_or_none(encoding.get('video_bitrate')),
+                    'abr': int_or_none(encoding.get('audio_bitrate')),
+                    'filesize': int_or_none(encoding.get('size_in_bytes')),
+                    'vcodec': encoding.get('video_codec'),
+                    'acodec': encoding.get('audio_codec'),
+                    'container': encoding.get('container_type'),
+                }
+                for f_url in (encoding_url, file_url):
+                    if not f_url:
+                        continue
+                    fmt = f.copy()
+                    rtmp = re.search(r'^(?P<url>rtmpe?://(?P<host>[^/]+)/(?P<app>.+))/(?P<playpath>mp[34]:.+)$', f_url)
+                    if rtmp:
+                        fmt.update({
+                            'url': rtmp.group('url'),
+                            'play_path': rtmp.group('playpath'),
+                            'app': rtmp.group('app'),
+                            'ext': 'flv',
+                            'format_id': 'rtmp',
+                        })
+                    else:
+                        fmt.update({
+                            'url': f_url,
+                            'format_id': 'http',
+                        })
+                    formats.append(fmt)
         self._sort_formats(formats)
 
+        title = media['title']
+
         subtitles = {}
         for closed_caption in media.get('closed_captions', []):
             sub_url = closed_caption.get('file')
@@ -140,7 +153,7 @@ class CuriosityStreamCollectionIE(CuriosityStreamBaseIE):
             'title': 'Curious Minds: The Internet',
             'description': 'How is the internet shaping our lives in the 21st Century?',
         },
-        'playlist_mincount': 17,
+        'playlist_mincount': 16,
     }, {
         'url': 'https://curiositystream.com/series/2',
         'only_matching': True,

From 281b8e34432d8dba9902be2c1eb77d3e6371cd73 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 7 Apr 2021 10:41:06 +0100
Subject: [PATCH 327/860] [jamendo] fix track extraction(closes #28686)

---
 youtube_dl/extractor/jamendo.py | 74 ++++++++++++++++++---------------
 1 file changed, 41 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/jamendo.py b/youtube_dl/extractor/jamendo.py
index 490efa8fb..1db7c64af 100644
--- a/youtube_dl/extractor/jamendo.py
+++ b/youtube_dl/extractor/jamendo.py
@@ -29,34 +29,51 @@ class JamendoIE(InfoExtractor):
             'id': '196219',
             'display_id': 'stories-from-emona-i',
             'ext': 'flac',
-            'title': 'Maya Filipič - Stories from Emona I',
-            'artist': 'Maya Filipič',
+            # 'title': 'Maya Filipič - Stories from Emona I',
+            'title': 'Stories from Emona I',
+            # 'artist': 'Maya Filipič',
             'track': 'Stories from Emona I',
             'duration': 210,
             'thumbnail': r're:^https?://.*\.jpg',
             'timestamp': 1217438117,
             'upload_date': '20080730',
+            'license': 'by-nc-nd',
+            'view_count': int,
+            'like_count': int,
+            'average_rating': int,
+            'tags': ['piano', 'peaceful', 'newage', 'strings', 'upbeat'],
         }
     }, {
         'url': 'https://licensing.jamendo.com/en/track/1496667/energetic-rock',
         'only_matching': True,
     }]
 
+    def _call_api(self, resource, resource_id):
+        path = '/api/%ss' % resource
+        rand = compat_str(random.random())
+        return self._download_json(
+            'https://www.jamendo.com' + path, resource_id, query={
+                'id[]': resource_id,
+            }, headers={
+                'X-Jam-Call': '$%s*%s~' % (hashlib.sha1((path + rand).encode()).hexdigest(), rand)
+            })[0]
+
     def _real_extract(self, url):
         track_id, display_id = self._VALID_URL_RE.match(url).groups()
-        webpage = self._download_webpage(
-            'https://www.jamendo.com/track/' + track_id, track_id)
-        models = self._parse_json(self._html_search_regex(
-            r"data-bundled-models='([^']+)",
-            webpage, 'bundled models'), track_id)
-        track = models['track']['models'][0]
+        # webpage = self._download_webpage(
+        #     'https://www.jamendo.com/track/' + track_id, track_id)
+        # models = self._parse_json(self._html_search_regex(
+        #     r"data-bundled-models='([^']+)",
+        #     webpage, 'bundled models'), track_id)
+        # track = models['track']['models'][0]
+        track = self._call_api('track', track_id)
         title = track_name = track['name']
-        get_model = lambda x: try_get(models, lambda y: y[x]['models'][0], dict) or {}
-        artist = get_model('artist')
-        artist_name = artist.get('name')
-        if artist_name:
-            title = '%s - %s' % (artist_name, title)
-        album = get_model('album')
+        # get_model = lambda x: try_get(models, lambda y: y[x]['models'][0], dict) or {}
+        # artist = get_model('artist')
+        # artist_name = artist.get('name')
+        # if artist_name:
+        #     title = '%s - %s' % (artist_name, title)
+        # album = get_model('album')
 
         formats = [{
             'url': 'https://%s.jamendo.com/?trackid=%s&format=%s&from=app-97dab294'
@@ -74,7 +91,7 @@ class JamendoIE(InfoExtractor):
 
         urls = []
         thumbnails = []
-        for _, covers in track.get('cover', {}).items():
+        for covers in (track.get('cover') or {}).values():
             for cover_id, cover_url in covers.items():
                 if not cover_url or cover_url in urls:
                     continue
@@ -88,13 +105,14 @@ class JamendoIE(InfoExtractor):
                 })
 
         tags = []
-        for tag in track.get('tags', []):
+        for tag in (track.get('tags') or []):
             tag_name = tag.get('name')
             if not tag_name:
                 continue
             tags.append(tag_name)
 
         stats = track.get('stats') or {}
+        license = track.get('licenseCC') or []
 
         return {
             'id': track_id,
@@ -103,11 +121,11 @@ class JamendoIE(InfoExtractor):
             'title': title,
             'description': track.get('description'),
             'duration': int_or_none(track.get('duration')),
-            'artist': artist_name,
+            # 'artist': artist_name,
             'track': track_name,
-            'album': album.get('name'),
+            # 'album': album.get('name'),
             'formats': formats,
-            'license': '-'.join(track.get('licenseCC', [])) or None,
+            'license': '-'.join(license) if license else None,
             'timestamp': int_or_none(track.get('dateCreated')),
             'view_count': int_or_none(stats.get('listenedAll')),
             'like_count': int_or_none(stats.get('favorited')),
@@ -116,9 +134,9 @@ class JamendoIE(InfoExtractor):
         }
 
 
-class JamendoAlbumIE(InfoExtractor):
+class JamendoAlbumIE(JamendoIE):
     _VALID_URL = r'https?://(?:www\.)?jamendo\.com/album/(?P<id>[0-9]+)'
-    _TEST = {
+    _TESTS = [{
         'url': 'https://www.jamendo.com/album/121486/duck-on-cover',
         'info_dict': {
             'id': '121486',
@@ -151,17 +169,7 @@ class JamendoAlbumIE(InfoExtractor):
         'params': {
             'playlistend': 2
         }
-    }
-
-    def _call_api(self, resource, resource_id):
-        path = '/api/%ss' % resource
-        rand = compat_str(random.random())
-        return self._download_json(
-            'https://www.jamendo.com' + path, resource_id, query={
-                'id[]': resource_id,
-            }, headers={
-                'X-Jam-Call': '$%s*%s~' % (hashlib.sha1((path + rand).encode()).hexdigest(), rand)
-            })[0]
+    }]
 
     def _real_extract(self, url):
         album_id = self._match_id(url)
@@ -169,7 +177,7 @@ class JamendoAlbumIE(InfoExtractor):
         album_name = album.get('name')
 
         entries = []
-        for track in album.get('tracks', []):
+        for track in (album.get('tracks') or []):
             track_id = track.get('id')
             if not track_id:
                 continue

From 006eea564d55130bb2e2ea7feb3a0e286d75d91f Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 7 Apr 2021 14:01:48 +0100
Subject: [PATCH 328/860] [cbssports] fix extraction(closes #28682)

---
 youtube_dl/extractor/cbssports.py  | 125 +++++++++++++++++++++++------
 youtube_dl/extractor/extractors.py |   6 +-
 2 files changed, 105 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/cbssports.py b/youtube_dl/extractor/cbssports.py
index 83b764762..a891c9a55 100644
--- a/youtube_dl/extractor/cbssports.py
+++ b/youtube_dl/extractor/cbssports.py
@@ -1,38 +1,113 @@
 from __future__ import unicode_literals
 
-from .cbs import CBSBaseIE
+import re
+
+# from .cbs import CBSBaseIE
+from .common import InfoExtractor
+from ..utils import (
+    int_or_none,
+    try_get,
+)
 
 
-class CBSSportsIE(CBSBaseIE):
-    _VALID_URL = r'https?://(?:www\.)?cbssports\.com/[^/]+/(?:video|news)/(?P<id>[^/?#&]+)'
-
+# class CBSSportsEmbedIE(CBSBaseIE):
+class CBSSportsEmbedIE(InfoExtractor):
+    IE_NAME = 'cbssports:embed'
+    _VALID_URL = r'''(?ix)https?://(?:(?:www\.)?cbs|embed\.247)sports\.com/player/embed.+?
+        (?:
+            ids%3D(?P<id>[\da-f]{8}-(?:[\da-f]{4}-){3}[\da-f]{12})|
+            pcid%3D(?P<pcid>\d+)
+        )'''
     _TESTS = [{
-        'url': 'https://www.cbssports.com/nba/video/donovan-mitchell-flashes-star-potential-in-game-2-victory-over-thunder/',
-        'info_dict': {
-            'id': '1214315075735',
-            'ext': 'mp4',
-            'title': 'Donovan Mitchell flashes star potential in Game 2 victory over Thunder',
-            'description': 'md5:df6f48622612c2d6bd2e295ddef58def',
-            'timestamp': 1524111457,
-            'upload_date': '20180419',
-            'uploader': 'CBSI-NEW',
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        }
+        'url': 'https://www.cbssports.com/player/embed/?args=player_id%3Db56c03a6-231a-4bbe-9c55-af3c8a8e9636%26ids%3Db56c03a6-231a-4bbe-9c55-af3c8a8e9636%26resizable%3D1%26autoplay%3Dtrue%26domain%3Dcbssports.com%26comp_ads_enabled%3Dfalse%26watchAndRead%3D0%26startTime%3D0%26env%3Dprod',
+        'only_matching': True,
     }, {
-        'url': 'https://www.cbssports.com/nba/news/nba-playoffs-2018-watch-76ers-vs-heat-game-3-series-schedule-tv-channel-online-stream/',
+        'url': 'https://embed.247sports.com/player/embed/?args=%3fplayer_id%3d1827823171591%26channel%3dcollege-football-recruiting%26pcid%3d1827823171591%26width%3d640%26height%3d360%26autoplay%3dTrue%26comp_ads_enabled%3dFalse%26uvpc%3dhttps%253a%252f%252fwww.cbssports.com%252fapi%252fcontent%252fvideo%252fconfig%252f%253fcfg%253duvp_247sports_v4%2526partner%253d247%26uvpc_m%3dhttps%253a%252f%252fwww.cbssports.com%252fapi%252fcontent%252fvideo%252fconfig%252f%253fcfg%253duvp_247sports_m_v4%2526partner_m%253d247_mobile%26utag%3d247sportssite%26resizable%3dTrue',
         'only_matching': True,
     }]
 
-    def _extract_video_info(self, filter_query, video_id):
-        return self._extract_feed_info('dJ5BDC', 'VxxJg8Ymh8sE', filter_query, video_id)
+    # def _extract_video_info(self, filter_query, video_id):
+    #     return self._extract_feed_info('dJ5BDC', 'VxxJg8Ymh8sE', filter_query, video_id)
 
+    def _real_extract(self, url):
+        uuid, pcid = re.match(self._VALID_URL, url).groups()
+        query = {'id': uuid} if uuid else {'pcid': pcid}
+        video = self._download_json(
+            'https://www.cbssports.com/api/content/video/',
+            uuid or pcid, query=query)[0]
+        video_id = video['id']
+        title = video['title']
+        metadata = video.get('metaData') or {}
+        # return self._extract_video_info('byId=%d' % metadata['mpxOutletId'], video_id)
+        # return self._extract_video_info('byGuid=' + metadata['mpxRefId'], video_id)
+
+        formats = self._extract_m3u8_formats(
+            metadata['files'][0]['url'], video_id, 'mp4',
+            'm3u8_native', m3u8_id='hls', fatal=False)
+        self._sort_formats(formats)
+
+        image = video.get('image')
+        thumbnails = None
+        if image:
+            image_path = image.get('path')
+            if image_path:
+                thumbnails = [{
+                    'url': image_path,
+                    'width': int_or_none(image.get('width')),
+                    'height': int_or_none(image.get('height')),
+                    'filesize': int_or_none(image.get('size')),
+                }]
+
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'thumbnails': thumbnails,
+            'description': video.get('description'),
+            'timestamp': int_or_none(try_get(video, lambda x: x['dateCreated']['epoch'])),
+            'duration': int_or_none(metadata.get('duration')),
+        }
+
+
+class CBSSportsBaseIE(InfoExtractor):
     def _real_extract(self, url):
         display_id = self._match_id(url)
         webpage = self._download_webpage(url, display_id)
-        video_id = self._search_regex(
-            [r'(?:=|%26)pcid%3D(\d+)', r'embedVideo(?:Container)?_(\d+)'],
-            webpage, 'video id')
-        return self._extract_video_info('byId=%s' % video_id, video_id)
+        iframe_url = self._search_regex(
+            r'<iframe[^>]+(?:data-)?src="(https?://[^/]+/player/embed[^"]+)"',
+            webpage, 'embed url')
+        return self.url_result(iframe_url, CBSSportsEmbedIE.ie_key())
+
+
+class CBSSportsIE(CBSSportsBaseIE):
+    IE_NAME = 'cbssports'
+    _VALID_URL = r'https?://(?:www\.)?cbssports\.com/[^/]+/video/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://www.cbssports.com/college-football/video/cover-3-stanford-spring-gleaning/',
+        'info_dict': {
+            'id': 'b56c03a6-231a-4bbe-9c55-af3c8a8e9636',
+            'ext': 'mp4',
+            'title': 'Cover 3: Stanford Spring Gleaning',
+            'description': 'The Cover 3 crew break down everything you need to know about the Stanford Cardinal this spring.',
+            'timestamp': 1617218398,
+            'upload_date': '20210331',
+            'duration': 502,
+        },
+    }]
+
+
+class TwentyFourSevenSportsIE(CBSSportsBaseIE):
+    IE_NAME = '247sports'
+    _VALID_URL = r'https?://(?:www\.)?247sports\.com/Video/(?:[^/?#&]+-)?(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://247sports.com/Video/2021-QB-Jake-Garcia-senior-highlights-through-five-games-10084854/',
+        'info_dict': {
+            'id': '4f1265cb-c3b5-44a8-bb1d-1914119a0ccc',
+            'ext': 'mp4',
+            'title': '2021 QB Jake Garcia senior highlights through five games',
+            'description': 'md5:8cb67ebed48e2e6adac1701e0ff6e45b',
+            'timestamp': 1607114223,
+            'upload_date': '20201204',
+            'duration': 208,
+        },
+    }]
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d5cd364e8..5ff9110b4 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -191,7 +191,11 @@ from .cbsnews import (
     CBSNewsIE,
     CBSNewsLiveVideoIE,
 )
-from .cbssports import CBSSportsIE
+from .cbssports import (
+    CBSSportsEmbedIE,
+    CBSSportsIE,
+    TwentyFourSevenSportsIE,
+)
 from .ccc import (
     CCCIE,
     CCCPlaylistIE,

From 545d6cb9d06a8bf32bcd24463c0fd25e650bb2c7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Thu, 8 Apr 2021 15:32:59 +0700
Subject: [PATCH 329/860] [pornhub] Extract DASH and HLS formats from get_media
 end point (closes #28698)

---
 youtube_dl/extractor/pornhub.py | 20 ++++++++++----------
 1 file changed, 10 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 2a7818e41..031454600 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -398,6 +398,16 @@ class PornHubIE(PornHubBaseIE):
         formats = []
 
         def add_format(format_url, height=None):
+            ext = determine_ext(format_url)
+            if ext == 'mpd':
+                formats.extend(self._extract_mpd_formats(
+                    format_url, video_id, mpd_id='dash', fatal=False))
+                return
+            if ext == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls', fatal=False))
+                return
             tbr = None
             mobj = re.search(r'(?P<height>\d+)[pP]?_(?P<tbr>\d+)[kK]', format_url)
             if mobj:
@@ -417,16 +427,6 @@ class PornHubIE(PornHubBaseIE):
                     r'/(\d{6}/\d{2})/', video_url, 'upload data', default=None)
                 if upload_date:
                     upload_date = upload_date.replace('/', '')
-            ext = determine_ext(video_url)
-            if ext == 'mpd':
-                formats.extend(self._extract_mpd_formats(
-                    video_url, video_id, mpd_id='dash', fatal=False))
-                continue
-            elif ext == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    video_url, video_id, 'mp4', entry_protocol='m3u8_native',
-                    m3u8_id='hls', fatal=False))
-                continue
             if '/video/get_media' in video_url:
                 medias = self._download_json(video_url, video_id, fatal=False)
                 if isinstance(medias, list):

From 27e5a4464d1d4c418d4937492e18a9d47d30fc50 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Thu, 8 Apr 2021 18:53:36 +0100
Subject: [PATCH 330/860] [mtv] Fix Viacom A/B Testing Video Player
 extraction(closes #28703)

---
 youtube_dl/extractor/mtv.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/mtv.py b/youtube_dl/extractor/mtv.py
index 600cf2d89..5a5205c0e 100644
--- a/youtube_dl/extractor/mtv.py
+++ b/youtube_dl/extractor/mtv.py
@@ -255,7 +255,9 @@ class MTVServicesInfoExtractor(InfoExtractor):
 
     @staticmethod
     def _extract_child_with_type(parent, t):
-        return next(c for c in parent['children'] if c.get('type') == t)
+        for c in parent['children']:
+            if c.get('type') == t:
+                return c
 
     def _extract_mgid(self, webpage):
         try:
@@ -286,7 +288,8 @@ class MTVServicesInfoExtractor(InfoExtractor):
             data = self._parse_json(self._search_regex(
                 r'__DATA__\s*=\s*({.+?});', webpage, 'data'), None)
             main_container = self._extract_child_with_type(data, 'MainContainer')
-            video_player = self._extract_child_with_type(main_container, 'VideoPlayer')
+            ab_testing = self._extract_child_with_type(main_container, 'ABTesting')
+            video_player = self._extract_child_with_type(ab_testing or main_container, 'VideoPlayer')
             mgid = video_player['props']['media']['video']['config']['uri']
 
         return mgid

From 1b0a13f33cfb3644cc718d35951ea85bb1905459 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 9 Apr 2021 02:09:52 +0700
Subject: [PATCH 331/860] [youtube:tab] Pass innertube context and
 x-goog-visitor-id header along with continuation requests (closes #28702)

---
 youtube_dl/extractor/youtube.py | 42 +++++++++++++++++++++------------
 1 file changed, 27 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6b4c7912c..79e47c919 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -306,7 +306,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         return self._parse_json(
             self._search_regex(
                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
-                default='{}'), video_id, fatal=False)
+                default='{}'), video_id, fatal=False) or {}
 
     def _extract_video(self, renderer):
         video_id = renderer['videoId']
@@ -2475,7 +2475,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             ctp = continuation_ep.get('clickTrackingParams')
             return YoutubeTabIE._build_continuation_query(continuation, ctp)
 
-    def _entries(self, tab, identity_token):
+    def _entries(self, tab, item_id, webpage):
         tab_content = try_get(tab, lambda x: x['content'], dict)
         if not tab_content:
             return
@@ -2535,26 +2535,37 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                 yield entry
             continuation = self._extract_continuation(rich_grid_renderer)
 
+        ytcfg = self._extract_ytcfg(item_id, webpage)
+        client_version = try_get(
+            ytcfg, lambda x: x['INNERTUBE_CLIENT_VERSION'], compat_str) or '2.20210407.08.00'
+
         headers = {
             'x-youtube-client-name': '1',
-            'x-youtube-client-version': '2.20201112.04.01',
+            'x-youtube-client-version': client_version,
             'content-type': 'application/json',
         }
+
+        context = try_get(ytcfg, lambda x: x['INNERTUBE_CONTEXT'], dict) or {
+            'client': {
+                'clientName': 'WEB',
+                'clientVersion': client_version,
+            }
+        }
+        visitor_data = try_get(context, lambda x: x['client']['visitorData'], compat_str)
+
+        identity_token = self._extract_identity_token(ytcfg, webpage)
         if identity_token:
             headers['x-youtube-identity-token'] = identity_token
 
         data = {
-            'context': {
-                'client': {
-                    'clientName': 'WEB',
-                    'clientVersion': '2.20201021.03.00',
-                }
-            },
+            'context': context,
         }
 
         for page_num in itertools.count(1):
             if not continuation:
                 break
+            if visitor_data:
+                headers['x-goog-visitor-id'] = visitor_data
             data['continuation'] = continuation['continuation']
             data['clickTracking'] = {
                 'clickTrackingParams': continuation['itct']
@@ -2579,6 +2590,9 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             if not response:
                 break
 
+            visitor_data = try_get(
+                response, lambda x: x['responseContext']['visitorData'], compat_str) or visitor_data
+
             continuation_contents = try_get(
                 response, lambda x: x['continuationContents'], dict)
             if continuation_contents:
@@ -2687,7 +2701,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                 alerts.append(text)
         return '\n'.join(alerts)
 
-    def _extract_from_tabs(self, item_id, webpage, data, tabs, identity_token):
+    def _extract_from_tabs(self, item_id, webpage, data, tabs):
         selected_tab = self._extract_selected_tab(tabs)
         renderer = try_get(
             data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
@@ -2712,7 +2726,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                 if renderer:
                     title = try_get(renderer, lambda x: x['hashtag']['simpleText'])
         playlist = self.playlist_result(
-            self._entries(selected_tab, identity_token),
+            self._entries(selected_tab, item_id, webpage),
             playlist_id=playlist_id, playlist_title=title,
             playlist_description=description)
         playlist.update(self._extract_uploader(data))
@@ -2736,8 +2750,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             self._playlist_entries(playlist), playlist_id=playlist_id,
             playlist_title=title)
 
-    def _extract_identity_token(self, webpage, item_id):
-        ytcfg = self._extract_ytcfg(item_id, webpage)
+    def _extract_identity_token(self, ytcfg, webpage):
         if ytcfg:
             token = try_get(ytcfg, lambda x: x['ID_TOKEN'], compat_str)
             if token:
@@ -2760,12 +2773,11 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                 return self.url_result(video_id, ie=YoutubeIE.ie_key(), video_id=video_id)
             self.to_screen('Downloading playlist %s - add --no-playlist to just download video %s' % (playlist_id, video_id))
         webpage = self._download_webpage(url, item_id)
-        identity_token = self._extract_identity_token(webpage, item_id)
         data = self._extract_yt_initial_data(item_id, webpage)
         tabs = try_get(
             data, lambda x: x['contents']['twoColumnBrowseResultsRenderer']['tabs'], list)
         if tabs:
-            return self._extract_from_tabs(item_id, webpage, data, tabs, identity_token)
+            return self._extract_from_tabs(item_id, webpage, data, tabs)
         playlist = try_get(
             data, lambda x: x['contents']['twoColumnWatchNextResults']['playlist']['playlist'], dict)
         if playlist:

From 4fb25ff5a3be5206bb72e5c4046715b1529fb2c7 Mon Sep 17 00:00:00 2001
From: Aaron Lipinski <kris.lipinski@gmail.com>
Date: Thu, 8 Apr 2021 19:59:36 +1200
Subject: [PATCH 332/860] [maoritv] Add new extractor(closes #24552)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/maoritv.py    | 31 ++++++++++++++++++++++++++++++
 2 files changed, 32 insertions(+)
 create mode 100644 youtube_dl/extractor/maoritv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 5ff9110b4..ac33cd996 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -640,6 +640,7 @@ from .mangomolo import (
     MangomoloLiveIE,
 )
 from .manyvids import ManyVidsIE
+from .maoritv import MaoriTVIE
 from .markiza import (
     MarkizaIE,
     MarkizaPageIE,
diff --git a/youtube_dl/extractor/maoritv.py b/youtube_dl/extractor/maoritv.py
new file mode 100644
index 000000000..0d23fec75
--- /dev/null
+++ b/youtube_dl/extractor/maoritv.py
@@ -0,0 +1,31 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+
+class MaoriTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?maoritelevision\.com/shows/(?:[^/]+/)+(?P<id>[^/?&#]+)'
+    _TEST = {
+        'url': 'https://www.maoritelevision.com/shows/korero-mai/S01E054/korero-mai-series-1-episode-54',
+        'md5': '5ade8ef53851b6a132c051b1cd858899',
+        'info_dict': {
+            'id': '4774724855001',
+            'ext': 'mp4',
+            'title': 'Kōrero Mai, Series 1 Episode 54',
+            'upload_date': '20160226',
+            'timestamp': 1456455018,
+            'description': 'md5:59bde32fd066d637a1a55794c56d8dcb',
+            'uploader_id': '1614493167001',
+        },
+    }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1614493167001/HJlhIQhQf_default/index.html?videoId=%s'
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url)
+        webpage = self._download_webpage(url, display_id)
+        brightcove_id = self._search_regex(
+            r'data-main-video-id=["\'](\d+)', webpage, 'brightcove id')
+        return self.url_result(
+            self.BRIGHTCOVE_URL_TEMPLATE % brightcove_id,
+            'BrightcoveNew', brightcove_id)

From 06159135ef148a6ddc632d0c89b90c937d5bb021 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Apr 2021 00:07:32 +0700
Subject: [PATCH 333/860] [youtube] Improve URL to extractor routing (closes
 #27572, closes #28335, closes #28742)

---
 youtube_dl/extractor/youtube.py | 35 ++++++++++++++++++++++-----------
 1 file changed, 23 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 79e47c919..4d7f3f837 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -46,6 +46,10 @@ from ..utils import (
 )
 
 
+def parse_qs(url):
+    return compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+
+
 class YoutubeBaseInfoExtractor(InfoExtractor):
     """Provide base functions for Youtube extractors"""
     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
@@ -413,16 +417,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                          |(?:www\.)?cleanvideosearch\.com/media/action/yt/watch\?videoId=
                          )
                      )?                                                       # all until now is optional -> you can pass the naked ID
-                     (?P<id>[0-9A-Za-z_-]{11})                                      # here is it! the YouTube video ID
-                     (?!.*?\blist=
-                        (?:
-                            %(playlist_id)s|                                  # combined list/video URLs are handled by the playlist IE
-                            WL                                                # WL are handled by the watch later IE
-                        )
-                     )
+                     (?P<id>[0-9A-Za-z_-]{11})                                # here is it! the YouTube video ID
                      (?(1).+)?                                                # if we found the ID, everything can follow
                      $""" % {
-        'playlist_id': YoutubeBaseInfoExtractor._PLAYLIST_ID_RE,
         'invidious': '|'.join(_INVIDIOUS_SITES),
     }
     _PLAYER_INFO_RE = (
@@ -1208,6 +1205,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '397': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
     }
 
+    @classmethod
+    def suitable(cls, url):
+        qs = parse_qs(url)
+        if qs.get('list', [None])[0]:
+            return False
+        return super(YoutubeIE, cls).suitable(url)
+
     def __init__(self, *args, **kwargs):
         super(YoutubeIE, self).__init__(*args, **kwargs)
         self._code_cache = {}
@@ -2275,6 +2279,9 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'title': '#cctv9',
         },
         'playlist_mincount': 350,
+    }, {
+        'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
+        'only_matching': True,
     }]
 
     @classmethod
@@ -2764,7 +2771,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
         url = compat_urlparse.urlunparse(
             compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com'))
         # Handle both video/playlist URLs
-        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+        qs = parse_qs(url)
         video_id = qs.get('v', [None])[0]
         playlist_id = qs.get('list', [None])[0]
         if video_id and playlist_id:
@@ -2860,12 +2867,16 @@ class YoutubePlaylistIE(InfoExtractor):
 
     @classmethod
     def suitable(cls, url):
-        return False if YoutubeTabIE.suitable(url) else super(
-            YoutubePlaylistIE, cls).suitable(url)
+        if YoutubeTabIE.suitable(url):
+            return False
+        qs = parse_qs(url)
+        if qs.get('v', [None])[0]:
+            return False
+        return super(YoutubePlaylistIE, cls).suitable(url)
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
-        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
+        qs = parse_qs(url)
         if not qs:
             qs = {'list': playlist_id}
         return self.url_result(

From 79e4ccfc4b395127bb3e5e957b20b04e75cba355 Mon Sep 17 00:00:00 2001
From: quyleanh <quyleanh@users.noreply.github.com>
Date: Sat, 17 Apr 2021 00:30:10 +0700
Subject: [PATCH 334/860] [pluralsight] Extend anti-throttling timeout (#28712)

---
 youtube_dl/extractor/pluralsight.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/pluralsight.py b/youtube_dl/extractor/pluralsight.py
index abd08bc28..2d63855df 100644
--- a/youtube_dl/extractor/pluralsight.py
+++ b/youtube_dl/extractor/pluralsight.py
@@ -393,7 +393,7 @@ query viewClip {
                 # To somewhat reduce the probability of these consequences
                 # we will sleep random amount of time before each call to ViewClip.
                 self._sleep(
-                    random.randint(2, 5), display_id,
+                    random.randint(5, 10), display_id,
                     '%(video_id)s: Waiting for %(timeout)s seconds to avoid throttling')
 
                 if not viewclip:

From d01e261a15abd779decae6e0858d8586f7a71621 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?C=C3=A1ssio=20=C3=81vila?= <cassioavila@yandex.com>
Date: Fri, 16 Apr 2021 14:31:34 -0300
Subject: [PATCH 335/860] [youtube] Add more invidious instances (#28706)

---
 youtube_dl/extractor/youtube.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4d7f3f837..7fa9b473a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -359,21 +359,28 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         r'(?:www\.)?invidious\.mastodon\.host',
         r'(?:www\.)?invidious\.zapashcanon\.fr',
         r'(?:www\.)?invidious\.kavin\.rocks',
+        r'(?:www\.)?invidious\.tinfoil-hat\.net',
+        r'(?:www\.)?invidious\.himiko\.cloud',
+        r'(?:www\.)?invidious\.reallyancient\.tech',
         r'(?:www\.)?invidious\.tube',
         r'(?:www\.)?invidiou\.site',
         r'(?:www\.)?invidious\.site',
         r'(?:www\.)?invidious\.xyz',
         r'(?:www\.)?invidious\.nixnet\.xyz',
+        r'(?:www\.)?invidious\.048596\.xyz',
         r'(?:www\.)?invidious\.drycat\.fr',
+        r'(?:www\.)?inv\.skyn3t\.in',
         r'(?:www\.)?tube\.poal\.co',
         r'(?:www\.)?tube\.connect\.cafe',
         r'(?:www\.)?vid\.wxzm\.sx',
         r'(?:www\.)?vid\.mint\.lgbt',
+        r'(?:www\.)?vid\.puffyan\.us',
         r'(?:www\.)?yewtu\.be',
         r'(?:www\.)?yt\.elukerio\.org',
         r'(?:www\.)?yt\.lelux\.fi',
         r'(?:www\.)?invidious\.ggc-project\.de',
         r'(?:www\.)?yt\.maisputain\.ovh',
+        r'(?:www\.)?ytprivate\.com',
         r'(?:www\.)?invidious\.13ad\.de',
         r'(?:www\.)?invidious\.toot\.koeln',
         r'(?:www\.)?invidious\.fdn\.fr',

From ea87ed8394127c4bf824688b8780eaf5a804e7a3 Mon Sep 17 00:00:00 2001
From: zraktvor <=>
Date: Sat, 10 Apr 2021 15:11:35 +0200
Subject: [PATCH 336/860] [youtube:tab] Detect series playlist on playlists
 page (closes #28723)

---
 youtube_dl/extractor/youtube.py | 24 +++++++++++++++++++++++-
 1 file changed, 23 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 7fa9b473a..581687d96 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2019,6 +2019,15 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'title': 'Игорь Клейнер - Playlists',
             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
         },
+    }, {
+        # playlists, series
+        'url': 'https://www.youtube.com/c/3blue1brown/playlists?view=50&sort=dd&shelf_id=3',
+        'playlist_mincount': 5,
+        'info_dict': {
+            'id': 'UCYO_jab_esuFRV4b17AJtAw',
+            'title': '3Blue1Brown - Playlists',
+            'description': 'md5:e1384e8a133307dd10edee76e875d62f',
+        },
     }, {
         # playlists, singlepage
         'url': 'https://www.youtube.com/user/ThirstForScience/playlists',
@@ -2311,7 +2320,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
 
     @staticmethod
     def _extract_grid_item_renderer(item):
-        for item_kind in ('Playlist', 'Video', 'Channel'):
+        for item_kind in ('Playlist', 'Video', 'Channel', 'Show'):
             renderer = item.get('grid%sRenderer' % item_kind)
             if renderer:
                 return renderer
@@ -2344,6 +2353,19 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                 yield self.url_result(
                     'https://www.youtube.com/channel/%s' % channel_id,
                     ie=YoutubeTabIE.ie_key(), video_title=title)
+            # show
+            if playlist_id is None:  # needs to check for playlist_id, or non-series playlists are recognized twice
+                show_playlist_url = try_get(
+                    renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
+                    compat_str)
+                if show_playlist_url:
+                    playlist_id = self._search_regex(r'/playlist\?list=([0-9a-zA-Z-_]+)', show_playlist_url,
+                                                     'playlist id', default=None)
+                    if playlist_id:
+                        title = try_get(renderer, lambda x: x['title']['simpleText'], compat_str)
+                        yield self.url_result(
+                            "https://www.youtube.com/playlist?list=%s" % playlist_id,
+                            ie=YoutubeTabIE.ie_key(), video_id=playlist_id, video_title=title)
 
     def _shelf_entries_from_content(self, shelf_renderer):
         content = shelf_renderer.get('content')

From 7c5239547928079513b65f62e4c84aea21ce76e6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Apr 2021 01:05:44 +0700
Subject: [PATCH 337/860] [youtube:tab] Improve grid extraction (closes #28725)

---
 youtube_dl/extractor/youtube.py | 38 ++++++++++++++++++---------------
 1 file changed, 21 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 581687d96..b6945570f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2320,10 +2320,13 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
 
     @staticmethod
     def _extract_grid_item_renderer(item):
-        for item_kind in ('Playlist', 'Video', 'Channel', 'Show'):
-            renderer = item.get('grid%sRenderer' % item_kind)
-            if renderer:
-                return renderer
+        assert isinstance(item, dict)
+        for key, renderer in item.items():
+            if not key.startswith('grid') or not key.endswith('Renderer'):
+                continue
+            if not isinstance(renderer, dict):
+                continue
+            return renderer
 
     def _grid_entries(self, grid_renderer):
         for item in grid_renderer['items']:
@@ -2333,7 +2336,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             if not isinstance(renderer, dict):
                 continue
             title = try_get(
-                renderer, lambda x: x['title']['runs'][0]['text'], compat_str)
+                renderer, (lambda x: x['title']['runs'][0]['text'],
+                           lambda x: x['title']['simpleText']), compat_str)
             # playlist
             playlist_id = renderer.get('playlistId')
             if playlist_id:
@@ -2341,10 +2345,12 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                     'https://www.youtube.com/playlist?list=%s' % playlist_id,
                     ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
                     video_title=title)
+                continue
             # video
             video_id = renderer.get('videoId')
             if video_id:
                 yield self._extract_video(renderer)
+                continue
             # channel
             channel_id = renderer.get('channelId')
             if channel_id:
@@ -2353,19 +2359,17 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                 yield self.url_result(
                     'https://www.youtube.com/channel/%s' % channel_id,
                     ie=YoutubeTabIE.ie_key(), video_title=title)
-            # show
-            if playlist_id is None:  # needs to check for playlist_id, or non-series playlists are recognized twice
-                show_playlist_url = try_get(
-                    renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
-                    compat_str)
-                if show_playlist_url:
-                    playlist_id = self._search_regex(r'/playlist\?list=([0-9a-zA-Z-_]+)', show_playlist_url,
-                                                     'playlist id', default=None)
-                    if playlist_id:
-                        title = try_get(renderer, lambda x: x['title']['simpleText'], compat_str)
+                continue
+            # generic endpoint URL support
+            ep_url = urljoin('https://www.youtube.com/', try_get(
+                renderer, lambda x: x['navigationEndpoint']['commandMetadata']['webCommandMetadata']['url'],
+                compat_str))
+            if ep_url:
+                for ie in (YoutubeTabIE, YoutubePlaylistIE, YoutubeIE):
+                    if ie.suitable(ep_url):
                         yield self.url_result(
-                            "https://www.youtube.com/playlist?list=%s" % playlist_id,
-                            ie=YoutubeTabIE.ie_key(), video_id=playlist_id, video_title=title)
+                            ep_url, ie=ie.ie_key(), video_id=ie._match_id(ep_url), video_title=title)
+                        break
 
     def _shelf_entries_from_content(self, shelf_renderer):
         content = shelf_renderer.get('content')

From 54558e0baa4d62a94af105cd1d7f8abcbd16b468 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Apr 2021 02:27:54 +0700
Subject: [PATCH 338/860] [youtube] Improve stretch extraction and fix
 stretched ratio calculation (closes #28769)

---
 youtube_dl/extractor/youtube.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index b6945570f..75751d5a6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -812,6 +812,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             },
             'skip': 'This video does not exist.',
         },
+        {
+            # Video with incomplete 'yt:stretch=16:'
+            'url': 'https://www.youtube.com/watch?v=FRhJzUSJbGI',
+            'only_matching': True,
+        },
         {
             # Video licensed under Creative Commons
             'url': 'https://www.youtube.com/watch?v=M4gD1WSo5mA',
@@ -1717,13 +1722,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 for m in re.finditer(self._meta_regex('og:video:tag'), webpage)]
         for keyword in keywords:
             if keyword.startswith('yt:stretch='):
-                w, h = keyword.split('=')[1].split(':')
-                w, h = int(w), int(h)
-                if w > 0 and h > 0:
-                    ratio = w / h
-                    for f in formats:
-                        if f.get('vcodec') != 'none':
-                            f['stretched_ratio'] = ratio
+                mobj = re.search(r'(\d+)\s*:\s*(\d+)', keyword)
+                if mobj:
+                    # NB: float is intentional for forcing float division
+                    w, h = (float(v) for v in mobj.groups())
+                    if w > 0 and h > 0:
+                        ratio = w / h
+                        for f in formats:
+                            if f.get('vcodec') != 'none':
+                                f['stretched_ratio'] = ratio
+                        break
 
         thumbnails = []
         for container in (video_details, microformat):

From a00a7e0cad3308d999599bf17df5d3e6aba502d8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Apr 2021 03:22:13 +0700
Subject: [PATCH 339/860] [utils] Add support for support for experimental HTTP
 response status code 308 Permanent Redirect (refs #27877, refs #28768)

---
 youtube_dl/utils.py | 62 ++++++++++++++++++++++++++++++++++++++++-----
 1 file changed, 56 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 8e4d144c9..538cc2b63 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -39,6 +39,7 @@ import zlib
 from .compat import (
     compat_HTMLParseError,
     compat_HTMLParser,
+    compat_HTTPError,
     compat_basestring,
     compat_chr,
     compat_cookiejar,
@@ -2879,12 +2880,61 @@ class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
 
 
 class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
-    if sys.version_info[0] < 3:
-        def redirect_request(self, req, fp, code, msg, headers, newurl):
-            # On python 2 urlh.geturl() may sometimes return redirect URL
-            # as byte string instead of unicode. This workaround allows
-            # to force it always return unicode.
-            return compat_urllib_request.HTTPRedirectHandler.redirect_request(self, req, fp, code, msg, headers, compat_str(newurl))
+    """YoutubeDL redirect handler
+
+    The code is based on HTTPRedirectHandler implementation from CPython [1].
+
+    This redirect handler solves two issues:
+     - ensures redirect URL is always unicode under python 2
+     - introduces support for experimental HTTP response status code
+       308 Permanent Redirect [2] used by some sites [3]
+
+    1. https://github.com/python/cpython/blob/master/Lib/urllib/request.py
+    2. https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/308
+    3. https://github.com/ytdl-org/youtube-dl/issues/28768
+    """
+
+    http_error_301 = http_error_303 = http_error_307 = http_error_308 = compat_urllib_request.HTTPRedirectHandler.http_error_302
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        """Return a Request or None in response to a redirect.
+
+        This is called by the http_error_30x methods when a
+        redirection response is received.  If a redirection should
+        take place, return a new Request to allow http_error_30x to
+        perform the redirect.  Otherwise, raise HTTPError if no-one
+        else should try to handle this url.  Return None if you can't
+        but another Handler might.
+        """
+        m = req.get_method()
+        if (not (code in (301, 302, 303, 307, 308) and m in ("GET", "HEAD")
+            or code in (301, 302, 303) and m == "POST")):
+            raise compat_HTTPError(req.full_url, code, msg, headers, fp)
+        # Strictly (according to RFC 2616), 301 or 302 in response to
+        # a POST MUST NOT cause a redirection without confirmation
+        # from the user (of urllib.request, in this case).  In practice,
+        # essentially all clients do redirect in this case, so we do
+        # the same.
+
+        # On python 2 urlh.geturl() may sometimes return redirect URL
+        # as byte string instead of unicode. This workaround allows
+        # to force it always return unicode.
+        if sys.version_info[0] < 3:
+            newurl = compat_str(newurl)
+
+        # Be conciliant with URIs containing a space.  This is mainly
+        # redundant with the more complete encoding done in http_error_302(),
+        # but it is kept for compatibility with other callers.
+        newurl = newurl.replace(' ', '%20')
+
+        CONTENT_HEADERS = ("content-length", "content-type")
+        # NB: don't use dict comprehension for python 2.6 compatibility
+        newheaders = dict((k, v) for k, v in req.headers.items()
+                           if k.lower() not in CONTENT_HEADERS)
+        return compat_urllib_request.Request(newurl,
+                       headers=newheaders,
+                       origin_req_host=req.origin_req_host,
+                       unverifiable=True)
 
 
 def extract_timezone(date_str):

From 30a3a4c70fdcad10ef1dc6c3402457a95fe1ae5c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Apr 2021 03:23:47 +0700
Subject: [PATCH 340/860] [lbry] Add support for HLS videos (closes #27877,
 closes #28768)

---
 youtube_dl/extractor/lbry.py | 30 +++++++++++++++++++++++++++++-
 1 file changed, 29 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/lbry.py b/youtube_dl/extractor/lbry.py
index ae43d56ea..cfd6b8393 100644
--- a/youtube_dl/extractor/lbry.py
+++ b/youtube_dl/extractor/lbry.py
@@ -120,6 +120,26 @@ class LBRYIE(LBRYBaseIE):
             'channel_url': 'https://lbry.tv/@LBRYFoundation:0ed629d2b9c601300cacf7eabe9da0be79010212',
             'vcodec': 'none',
         }
+    }, {
+        # HLS
+        'url': 'https://odysee.com/@gardeningincanada:b/plants-i-will-never-grow-again.-the:e',
+        'md5': 'fc82f45ea54915b1495dd7cb5cc1289f',
+        'info_dict': {
+            'id': 'e51671357333fe22ae88aad320bde2f6f96b1410',
+            'ext': 'mp4',
+            'title': 'PLANTS I WILL NEVER GROW AGAIN. THE BLACK LIST PLANTS FOR A CANADIAN GARDEN | Gardening in Canada 🍁',
+            'description': 'md5:9c539c6a03fb843956de61a4d5288d5e',
+            'timestamp': 1618254123,
+            'upload_date': '20210412',
+            'release_timestamp': 1618254002,
+            'release_date': '20210412',
+            'tags': list,
+            'duration': 554,
+            'channel': 'Gardening In Canada',
+            'channel_id': 'b8be0e93b423dad221abe29545fbe8ec36e806bc',
+            'channel_url': 'https://odysee.com/@gardeningincanada:b8be0e93b423dad221abe29545fbe8ec36e806bc',
+            'formats': 'mincount:3',
+        }
     }, {
         'url': 'https://odysee.com/@BrodieRobertson:5/apple-is-tracking-everything-you-do-on:e',
         'only_matching': True,
@@ -163,10 +183,18 @@ class LBRYIE(LBRYBaseIE):
         streaming_url = self._call_api_proxy(
             'get', claim_id, {'uri': uri}, 'streaming url')['streaming_url']
         info = self._parse_stream(result, url)
+        urlh = self._request_webpage(
+            streaming_url, display_id, note='Downloading streaming redirect url info')
+        if determine_ext(urlh.geturl()) == 'm3u8':
+            info['formats'] = self._extract_m3u8_formats(
+                urlh.geturl(), display_id, 'mp4', entry_protocol='m3u8_native',
+                m3u8_id='hls')
+            self._sort_formats(info['formats'])
+        else:
+            info['url'] = streaming_url
         info.update({
             'id': claim_id,
             'title': title,
-            'url': streaming_url,
         })
         return info
 

From cfee2dfe83c5593d46bd0c8e8ce6a3d8c6e42db7 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Apr 2021 03:32:04 +0700
Subject: [PATCH 341/860] [utils] PEP 8

---
 youtube_dl/utils.py | 11 +++++------
 1 file changed, 5 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 538cc2b63..e722eed58 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2908,7 +2908,7 @@ class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
         """
         m = req.get_method()
         if (not (code in (301, 302, 303, 307, 308) and m in ("GET", "HEAD")
-            or code in (301, 302, 303) and m == "POST")):
+                 or code in (301, 302, 303) and m == "POST")):
             raise compat_HTTPError(req.full_url, code, msg, headers, fp)
         # Strictly (according to RFC 2616), 301 or 302 in response to
         # a POST MUST NOT cause a redirection without confirmation
@@ -2930,11 +2930,10 @@ class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
         CONTENT_HEADERS = ("content-length", "content-type")
         # NB: don't use dict comprehension for python 2.6 compatibility
         newheaders = dict((k, v) for k, v in req.headers.items()
-                           if k.lower() not in CONTENT_HEADERS)
-        return compat_urllib_request.Request(newurl,
-                       headers=newheaders,
-                       origin_req_host=req.origin_req_host,
-                       unverifiable=True)
+                          if k.lower() not in CONTENT_HEADERS)
+        return compat_urllib_request.Request(
+            newurl, headers=newheaders, origin_req_host=req.origin_req_host,
+            unverifiable=True)
 
 
 def extract_timezone(date_str):

From f20b505b46e6654464635ca4afb0f37e1f14c57b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Apr 2021 03:47:00 +0700
Subject: [PATCH 342/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 22b4fa67d..c249412e2 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,28 @@
+version <unreleased>
+
+Core
++ [utils] Add support for experimental HTTP response status code
+  308 Permanent Redirect (#27877, #28768)
+
+Extractors
++ [lbry] Add support for HLS videos (#27877, #28768)
+* [youtube] Fix stretched ratio calculation
+* [youtube] Improve stretch extraction (#28769)
+* [youtube:tab] Improve grid extraction (#28725)
++ [youtube:tab] Detect series playlist on playlists page (#28723)
++ [youtube] Add more invidious instances (#28706)
+* [pluralsight] Extend anti-throttling timeout (#28712)
+* [youtube] Improve URL to extractor routing (#27572, #28335, #28742)
++ [maoritv] Add support for maoritelevision.com (#24552)
++ [youtube:tab] Pass innertube context and x-goog-visitor-id header along with
+  continuation requests (#28702)
+* [mtv] Fix Viacom A/B Testing Video Player extraction (#28703)
++ [pornhub] Extract DASH and HLS formats from get_media end point (#28698)
+* [cbssports] Fix extraction (#28682)
+* [jamendo] Fix track extraction (#28686)
+* [curiositystream] Fix format extraction (#26845, #28668)
+
+
 version 2021.04.07
 
 Core

From 596b26606cfe20aa9f776ac0658b6bfb1ea95397 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Apr 2021 03:50:09 +0700
Subject: [PATCH 343/860] release 2021.04.17

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 docs/supportedsites.md                           | 5 ++++-
 youtube_dl/version.py                            | 2 +-
 8 files changed, 18 insertions(+), 15 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index febbd2344..e2a89d5c2 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.07. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.04.07**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.17**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.04.07
+ [debug] youtube-dl version 2021.04.17
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index d7296d0a9..4d7abd775 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.07. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.04.07**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.17**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 92e616a1a..d8dce6fd4 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.07. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.04.07**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.17**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index b55739f6c..d95ee291a 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.07. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.04.07**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.17**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.04.07
+ [debug] youtube-dl version 2021.04.17
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index dbdb8356a..ac5dd2f27 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.07. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.04.07**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.17**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index c249412e2..45d5c2ebf 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.04.17
 
 Core
 + [utils] Add support for experimental HTTP response status code
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index ff9177a2c..a23da1a31 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -3,6 +3,7 @@
  - **20min**
  - **220.ro**
  - **23video**
+ - **247sports**
  - **24video**
  - **3qsdn**: 3Q SDN
  - **3sat**
@@ -160,7 +161,8 @@
  - **cbsnews**: CBS News
  - **cbsnews:embed**
  - **cbsnews:livevideo**: CBS News Live Videos
- - **CBSSports**
+ - **cbssports**
+ - **cbssports:embed**
  - **CCMA**
  - **CCTV**: 央视网
  - **CDA**
@@ -490,6 +492,7 @@
  - **mangomolo:live**
  - **mangomolo:video**
  - **ManyVids**
+ - **MaoriTV**
  - **Markiza**
  - **MarkizaPage**
  - **massengeschmack.tv**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index a6b1b8dce..2b041d593 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.04.07'
+__version__ = '2021.04.17'

From 9f6c03a00602eb1119e43a522cf50682f6d6a6dd Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 17 Apr 2021 05:05:31 +0700
Subject: [PATCH 344/860] [cbsnews] Fix extraction for python <3.6 (closes
 #23359)

---
 youtube_dl/extractor/cbsnews.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/cbsnews.py b/youtube_dl/extractor/cbsnews.py
index 345debcf0..1285ed65e 100644
--- a/youtube_dl/extractor/cbsnews.py
+++ b/youtube_dl/extractor/cbsnews.py
@@ -26,7 +26,7 @@ class CBSNewsEmbedIE(CBSIE):
     def _real_extract(self, url):
         item = self._parse_json(zlib.decompress(compat_b64decode(
             compat_urllib_parse_unquote(self._match_id(url))),
-            -zlib.MAX_WBITS), None)['video']['items'][0]
+            -zlib.MAX_WBITS).decode('utf-8'), None)['video']['items'][0]
         return self._extract_video_info(item['mpxRefId'], 'cbsnews')
 
 

From 41920fc80e4fe4a8996aeb31a04826a5a2534814 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 20 Apr 2021 20:51:55 +0100
Subject: [PATCH 345/860] [bbc] Extract description and timestamp from
 __INITIAL_DATA__ (#28774)

---
 youtube_dl/extractor/bbc.py | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index e8d000bbb..71ea25881 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -11,6 +11,7 @@ from ..compat import (
     compat_etree_Element,
     compat_HTTPError,
     compat_parse_qs,
+    compat_str,
     compat_urllib_parse_urlparse,
     compat_urlparse,
 )
@@ -25,8 +26,10 @@ from ..utils import (
     js_to_json,
     parse_duration,
     parse_iso8601,
+    strip_or_none,
     try_get,
     unescapeHTML,
+    unified_timestamp,
     url_or_none,
     urlencode_postdata,
     urljoin,
@@ -761,8 +764,17 @@ class BBCIE(BBCCoUkIE):
         'only_matching': True,
     }, {
         # custom redirection to www.bbc.com
+        # also, video with window.__INITIAL_DATA__
         'url': 'http://www.bbc.co.uk/news/science-environment-33661876',
-        'only_matching': True,
+        'info_dict': {
+            'id': 'p02xzws1',
+            'ext': 'mp4',
+            'title': "Pluto may have 'nitrogen glaciers'",
+            'description': "Pluto could have glaciers of nitrogen ice, new photographs from Nasa's New Horizons probe suggest.",
+            'thumbnail': r're:https?://.+/.+\.jpg',
+            'timestamp': 1437785037,
+            'upload_date': '20150725',
+        },
     }, {
         # single video article embedded with data-media-vpid
         'url': 'http://www.bbc.co.uk/sport/rowing/35908187',
@@ -1164,12 +1176,23 @@ class BBCIE(BBCCoUkIE):
                         continue
                     formats, subtitles = self._download_media_selector(item_id)
                     self._sort_formats(formats)
+                    item_desc = try_get(
+                        media,
+                        lambda x: x['summary']['blocks'][0]['model']['text'],
+                        compat_str)
+                    item_time = None
+                    for meta in try_get(media, lambda x: x['metadata']['items'], list) or []:
+                        if try_get(meta, lambda x: x['label']) == 'Published':
+                            item_time = unified_timestamp(meta.get('timestamp'))
+                            break
                     entries.append({
                         'id': item_id,
                         'title': item_title,
                         'thumbnail': item.get('holdingImageUrl'),
                         'formats': formats,
                         'subtitles': subtitles,
+                        'timestamp': item_time,
+                        'description': strip_or_none(item_desc),
                     })
             for resp in (initial_data.get('data') or {}).values():
                 name = resp.get('name')

From dab83a25972e0dbcc69583bf78d2a992f581563d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Apr 2021 03:00:56 +0700
Subject: [PATCH 346/860] [bbc] Extract full description from __INITIAL_DATA__
 (refs #28774)

---
 youtube_dl/extractor/bbc.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 71ea25881..247d982ce 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -770,7 +770,7 @@ class BBCIE(BBCCoUkIE):
             'id': 'p02xzws1',
             'ext': 'mp4',
             'title': "Pluto may have 'nitrogen glaciers'",
-            'description': "Pluto could have glaciers of nitrogen ice, new photographs from Nasa's New Horizons probe suggest.",
+            'description': 'md5:6a95b593f528d7a5f2605221bc56912f',
             'thumbnail': r're:https?://.+/.+\.jpg',
             'timestamp': 1437785037,
             'upload_date': '20150725',
@@ -1176,10 +1176,16 @@ class BBCIE(BBCCoUkIE):
                         continue
                     formats, subtitles = self._download_media_selector(item_id)
                     self._sort_formats(formats)
-                    item_desc = try_get(
-                        media,
-                        lambda x: x['summary']['blocks'][0]['model']['text'],
-                        compat_str)
+                    item_desc = None
+                    blocks = try_get(media, lambda x: x['summary']['blocks'], list)
+                    if blocks:
+                        summary = []
+                        for block in blocks:
+                            text = try_get(block, lambda x: x['model']['text'], compat_str)
+                            if text:
+                                summary.append(text)
+                        if summary:
+                            item_desc = '\n\n'.join(summary)
                     item_time = None
                     for meta in try_get(media, lambda x: x['metadata']['items'], list) or []:
                         if try_get(meta, lambda x: x['label']) == 'Published':

From 32290307a45260885b2210aa3c2a57e64abf8c34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Apr 2021 03:56:04 +0700
Subject: [PATCH 347/860] [youtube] Fix lazy extractors (closes #28780)

---
 youtube_dl/extractor/youtube.py | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 75751d5a6..c16dc7ab8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1219,6 +1219,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     @classmethod
     def suitable(cls, url):
+        # Hack for lazy extractors until more generic solution is implemented
+        # (see #28780)
+        from .youtube import parse_qs
         qs = parse_qs(url)
         if qs.get('list', [None])[0]:
             return False
@@ -2910,6 +2913,9 @@ class YoutubePlaylistIE(InfoExtractor):
     def suitable(cls, url):
         if YoutubeTabIE.suitable(url):
             return False
+        # Hack for lazy extractors until more generic solution is implemented
+        # (see #28780)
+        from .youtube import parse_qs
         qs = parse_qs(url)
         if qs.get('v', [None])[0]:
             return False

From 5ad69d3d0e7d1d8d15a1f2497d602b1b91fcc74a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Apr 2021 04:45:13 +0700
Subject: [PATCH 348/860] [test_youtube_misc] Move YoutubeIE.extract_id test
 into separate module

---
 test/test_all_urls.py     |  9 ---------
 test/test_youtube_misc.py | 26 ++++++++++++++++++++++++++
 2 files changed, 26 insertions(+), 9 deletions(-)
 create mode 100644 test/test_youtube_misc.py

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index df6d81b5d..365b66bad 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -70,15 +70,6 @@ class TestAllURLsMatching(unittest.TestCase):
     #     self.assertMatch('http://www.youtube.com/results?search_query=making+mustard', ['youtube:search_url'])
     #     self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url'])
 
-    def test_youtube_extract(self):
-        assertExtractId = lambda url, id: self.assertEqual(YoutubeIE.extract_id(url), id)
-        assertExtractId('http://www.youtube.com/watch?&v=BaW_jenozKc', 'BaW_jenozKc')
-        assertExtractId('https://www.youtube.com/watch?&v=BaW_jenozKc', 'BaW_jenozKc')
-        assertExtractId('https://www.youtube.com/watch?feature=player_embedded&v=BaW_jenozKc', 'BaW_jenozKc')
-        assertExtractId('https://www.youtube.com/watch_popup?v=BaW_jenozKc', 'BaW_jenozKc')
-        assertExtractId('http://www.youtube.com/watch?v=BaW_jenozKcsharePLED17F32AD9753930', 'BaW_jenozKc')
-        assertExtractId('BaW_jenozKc', 'BaW_jenozKc')
-
     def test_facebook_matching(self):
         self.assertTrue(FacebookIE.suitable('https://www.facebook.com/Shiniknoh#!/photo.php?v=10153317450565268'))
         self.assertTrue(FacebookIE.suitable('https://www.facebook.com/cindyweather?fref=ts#!/photo.php?v=10152183998945793'))
diff --git a/test/test_youtube_misc.py b/test/test_youtube_misc.py
new file mode 100644
index 000000000..e18e71101
--- /dev/null
+++ b/test/test_youtube_misc.py
@@ -0,0 +1,26 @@
+#!/usr/bin/env python
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+from youtube_dl.extractor import YoutubeIE
+
+
+class TestYoutubeMisc(unittest.TestCase):
+    def test_youtube_extract(self):
+        assertExtractId = lambda url, id: self.assertEqual(YoutubeIE.extract_id(url), id)
+        assertExtractId('http://www.youtube.com/watch?&v=BaW_jenozKc', 'BaW_jenozKc')
+        assertExtractId('https://www.youtube.com/watch?&v=BaW_jenozKc', 'BaW_jenozKc')
+        assertExtractId('https://www.youtube.com/watch?feature=player_embedded&v=BaW_jenozKc', 'BaW_jenozKc')
+        assertExtractId('https://www.youtube.com/watch_popup?v=BaW_jenozKc', 'BaW_jenozKc')
+        assertExtractId('http://www.youtube.com/watch?v=BaW_jenozKcsharePLED17F32AD9753930', 'BaW_jenozKc')
+        assertExtractId('BaW_jenozKc', 'BaW_jenozKc')
+
+
+if __name__ == '__main__':
+    unittest.main()

From c4a451bcdd2b743fdb96fcbae261c86ed91022ba Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Apr 2021 04:47:29 +0700
Subject: [PATCH 349/860] [test_execution] Add test for lazy extractors (refs
 #28780)

---
 test/test_execution.py | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/test/test_execution.py b/test/test_execution.py
index 11661bb68..32948d93e 100644
--- a/test/test_execution.py
+++ b/test/test_execution.py
@@ -39,6 +39,16 @@ class TestExecution(unittest.TestCase):
         _, stderr = p.communicate()
         self.assertFalse(stderr)
 
+    def test_lazy_extractors(self):
+        try:
+            subprocess.check_call([sys.executable, 'devscripts/make_lazy_extractors.py', 'youtube_dl/extractor/lazy_extractors.py'], cwd=rootDir, stdout=_DEV_NULL)
+            subprocess.check_call([sys.executable, 'test/test_all_urls.py'], cwd=rootDir, stdout=_DEV_NULL)
+        finally:
+            try:
+                os.remove('youtube_dl/extractor/lazy_extractors.py')
+            except (IOError, OSError):
+                pass
+
 
 if __name__ == '__main__':
     unittest.main()

From ac19c3ac8035fbf9369fd4bd336c9045d4eeafa3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Apr 2021 05:35:39 +0700
Subject: [PATCH 350/860] [go] Improve video id extraction (closes #25207,
 closes #25216, closes #26058)

---
 youtube_dl/extractor/go.py | 46 +++++++++++++++++++++++++++++++-------
 1 file changed, 38 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/go.py b/youtube_dl/extractor/go.py
index 0d731e90a..878ba14e6 100644
--- a/youtube_dl/extractor/go.py
+++ b/youtube_dl/extractor/go.py
@@ -4,10 +4,12 @@ from __future__ import unicode_literals
 import re
 
 from .adobepass import AdobePassIE
+from ..compat import compat_str
 from ..utils import (
     int_or_none,
     determine_ext,
     parse_age_limit,
+    try_get,
     urlencode_postdata,
     ExtractorError,
 )
@@ -116,6 +118,18 @@ class GoIE(AdobePassIE):
             # m3u8 download
             'skip_download': True,
         },
+    }, {
+        'url': 'https://abc.com/shows/modern-family/episode-guide/season-01/101-pilot',
+        'info_dict': {
+            'id': 'VDKA22600213',
+            'ext': 'mp4',
+            'title': 'Pilot',
+            'description': 'md5:74306df917cfc199d76d061d66bebdb4',
+        },
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
     }, {
         'url': 'http://abc.go.com/shows/the-catch/episode-guide/season-01/10-the-wedding',
         'only_matching': True,
@@ -149,14 +163,30 @@ class GoIE(AdobePassIE):
         brand = site_info.get('brand')
         if not video_id or not site_info:
             webpage = self._download_webpage(url, display_id or video_id)
-            video_id = self._search_regex(
-                (
-                    # There may be inner quotes, e.g. data-video-id="'VDKA3609139'"
-                    # from http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood
-                    r'data-video-id=["\']*(VDKA\w+)',
-                    # https://abc.com/shows/the-rookie/episode-guide/season-02/03-the-bet
-                    r'\b(?:video)?id["\']\s*:\s*["\'](VDKA\w+)'
-                ), webpage, 'video id', default=video_id)
+            data = self._parse_json(
+                self._search_regex(
+                    r'["\']__abc_com__["\']\s*\]\s*=\s*({.+?})\s*;', webpage,
+                    'data', default='{}'),
+                display_id or video_id, fatal=False)
+            # https://abc.com/shows/modern-family/episode-guide/season-01/101-pilot
+            layout = try_get(data, lambda x: x['page']['content']['video']['layout'], dict)
+            video_id = None
+            if layout:
+                video_id = try_get(
+                    layout,
+                    (lambda x: x['videoid'], lambda x: x['video']['id']),
+                    compat_str)
+            if not video_id:
+                video_id = self._search_regex(
+                    (
+                        # There may be inner quotes, e.g. data-video-id="'VDKA3609139'"
+                        # from http://freeform.go.com/shows/shadowhunters/episodes/season-2/1-this-guilty-blood
+                        r'data-video-id=["\']*(VDKA\w+)',
+                        # page.analytics.videoIdCode
+                        r'\bvideoIdCode["\']\s*:\s*["\']((?:vdka|VDKA)\w+)',
+                        # https://abc.com/shows/the-rookie/episode-guide/season-02/03-the-bet
+                        r'\b(?:video)?id["\']\s*:\s*["\'](VDKA\w+)'
+                    ), webpage, 'video id', default=video_id)
             if not site_info:
                 brand = self._search_regex(
                     (r'data-brand=\s*["\']\s*(\d+)',

From 7e8b3f9439ebefb3a3a4e5da9c0bd2b595976438 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 21 Apr 2021 05:37:51 +0700
Subject: [PATCH 351/860] [youtube] Remove unused code

---
 youtube_dl/extractor/youtube.py | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c16dc7ab8..0c52e5a8b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -65,11 +65,6 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM)'
 
-    def _ids_to_results(self, ids):
-        return [
-            self.url_result(vid_id, 'Youtube', video_id=vid_id)
-            for vid_id in ids]
-
     def _login(self):
         """
         Attempt to log in to YouTube.

From 0db79d8181c1c3ebf74b9b6d38262c8dcaaf0f4f Mon Sep 17 00:00:00 2001
From: The Hatsune Daishi <nao20010128@gmail.com>
Date: Sat, 24 Apr 2021 20:58:03 +0900
Subject: [PATCH 352/860] [tver] Redirect all downloads to Brightcove (#28849)

---
 youtube_dl/extractor/tver.py | 37 +++++++++++-------------------------
 1 file changed, 11 insertions(+), 26 deletions(-)

diff --git a/youtube_dl/extractor/tver.py b/youtube_dl/extractor/tver.py
index a54f49319..a4a30b1e6 100644
--- a/youtube_dl/extractor/tver.py
+++ b/youtube_dl/extractor/tver.py
@@ -9,7 +9,6 @@ from ..utils import (
     int_or_none,
     remove_start,
     smuggle_url,
-    strip_or_none,
     try_get,
 )
 
@@ -45,32 +44,18 @@ class TVerIE(InfoExtractor):
             query={'token': self._TOKEN})['main']
         p_id = main['publisher_id']
         service = remove_start(main['service'], 'ts_')
-        info = {
+
+        r_id = main['reference_id']
+        if service not in ('tx', 'russia2018', 'sebare2018live', 'gorin'):
+            r_id = 'ref:' + r_id
+        bc_url = smuggle_url(
+            self.BRIGHTCOVE_URL_TEMPLATE % (p_id, r_id),
+            {'geo_countries': ['JP']})
+
+        return {
             '_type': 'url_transparent',
             'description': try_get(main, lambda x: x['note'][0]['text'], compat_str),
             'episode_number': int_or_none(try_get(main, lambda x: x['ext']['episode_number'])),
+            'url': bc_url,
+            'ie_key': 'BrightcoveNew',
         }
-
-        if service == 'cx':
-            title = main['title']
-            subtitle = strip_or_none(main.get('subtitle'))
-            if subtitle:
-                title += ' - ' + subtitle
-            info.update({
-                'title': title,
-                'url': 'https://i.fod.fujitv.co.jp/plus7/web/%s/%s.html' % (p_id[:4], p_id),
-                'ie_key': 'FujiTVFODPlus7',
-            })
-        else:
-            r_id = main['reference_id']
-            if service not in ('tx', 'russia2018', 'sebare2018live', 'gorin'):
-                r_id = 'ref:' + r_id
-            bc_url = smuggle_url(
-                self.BRIGHTCOVE_URL_TEMPLATE % (p_id, r_id),
-                {'geo_countries': ['JP']})
-            info.update({
-                'url': bc_url,
-                'ie_key': 'BrightcoveNew',
-            })
-
-        return info

From c6ab79299034492c5af9dd29b0c49585e4efc4cd Mon Sep 17 00:00:00 2001
From: catboy <79282513+catboy-oss@users.noreply.github.com>
Date: Sat, 24 Apr 2021 12:10:35 +0000
Subject: [PATCH 353/860] [medaltv] Fix extraction (#28807)

numeric clip ids are no longer used by medal, and integer user ids are now sent as strings.
---
 youtube_dl/extractor/medaltv.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/medaltv.py b/youtube_dl/extractor/medaltv.py
index 1603b55f6..ef2283dea 100644
--- a/youtube_dl/extractor/medaltv.py
+++ b/youtube_dl/extractor/medaltv.py
@@ -15,32 +15,32 @@ from ..utils import (
 
 
 class MedalTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?medal\.tv/clips/(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?medal\.tv/clips/(?P<id>[a-zA-Z0-9]+)'
     _TESTS = [{
-        'url': 'https://medal.tv/clips/34934644/3Is9zyGMoBMr',
+        'url': 'https://medal.tv/clips/2mA60jWAGQCBH',
         'md5': '7b07b064331b1cf9e8e5c52a06ae68fa',
         'info_dict': {
-            'id': '34934644',
+            'id': '2mA60jWAGQCBH',
             'ext': 'mp4',
             'title': 'Quad Cold',
             'description': 'Medal,https://medal.tv/desktop/',
             'uploader': 'MowgliSB',
             'timestamp': 1603165266,
             'upload_date': '20201020',
-            'uploader_id': 10619174,
+            'uploader_id': '10619174',
         }
     }, {
-        'url': 'https://medal.tv/clips/36787208',
+        'url': 'https://medal.tv/clips/2um24TWdty0NA',
         'md5': 'b6dc76b78195fff0b4f8bf4a33ec2148',
         'info_dict': {
-            'id': '36787208',
+            'id': '2um24TWdty0NA',
             'ext': 'mp4',
             'title': 'u tk me i tk u bigger',
             'description': 'Medal,https://medal.tv/desktop/',
             'uploader': 'Mimicc',
             'timestamp': 1605580939,
             'upload_date': '20201117',
-            'uploader_id': 5156321,
+            'uploader_id': '5156321',
         }
     }]
 

From 999329cf6b29878d054c6ccdd24573489a2886d5 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 24 Apr 2021 23:52:16 +0700
Subject: [PATCH 354/860] [workflows/ci.yml] Fix install nose for Jython

---
 .github/workflows/ci.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a9dc47a71..97b50aedc 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -53,7 +53,14 @@ jobs:
         java -jar jython-installer.jar -s -d "$HOME/jython"
         echo "$HOME/jython/bin" >> $GITHUB_PATH
     - name: Install nose
+      if: ${{ matrix.python-impl != 'jython' }}
       run: pip install nose
+    - name: Install nose (Jython)
+      if: ${{ matrix.python-impl == 'jython' }}
+      # Working around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb)
+      run: |
+        wget https://files.pythonhosted.org/packages/99/4f/13fb671119e65c4dce97c60e67d3fd9e6f7f809f2b307e2611f4701205cb/nose-1.3.7-py2-none-any.whl
+        pip install nose-1.3.7-py2-none-any.whl
     - name: Run tests
       continue-on-error: ${{ matrix.ytdl-test-set == 'download' || matrix.python-impl == 'jython' }}
       env:

From 57eaaff5cf1ae63d4e3ae89f301c0f9b3e86bb55 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 25 Apr 2021 22:52:28 +0700
Subject: [PATCH 355/860] [francetvinfo] Improve video id extraction (closes
 #28792)

---
 youtube_dl/extractor/francetv.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/francetv.py b/youtube_dl/extractor/francetv.py
index 7cc88bf18..e4ec2e200 100644
--- a/youtube_dl/extractor/francetv.py
+++ b/youtube_dl/extractor/francetv.py
@@ -383,6 +383,10 @@ class FranceTVInfoIE(FranceTVBaseInfoExtractor):
     }, {
         'url': 'http://france3-regions.francetvinfo.fr/limousin/emissions/jt-1213-limousin',
         'only_matching': True,
+    }, {
+        # "<figure id=" pattern (#28792)
+        'url': 'https://www.francetvinfo.fr/culture/patrimoine/incendie-de-notre-dame-de-paris/notre-dame-de-paris-de-l-incendie-de-la-cathedrale-a-sa-reconstruction_4372291.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -400,7 +404,7 @@ class FranceTVInfoIE(FranceTVBaseInfoExtractor):
             (r'player\.load[^;]+src:\s*["\']([^"\']+)',
              r'id-video=([^@]+@[^"]+)',
              r'<a[^>]+href="(?:https?:)?//videos\.francetv\.fr/video/([^@]+@[^"]+)"',
-             r'data-id=["\']([\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'),
+             r'(?:data-id|<figure[^<]+\bid)=["\']([\da-f]{8}-[\da-f]{4}-[\da-f]{4}-[\da-f]{4}-[\da-f]{12})'),
             webpage, 'video id')
 
         return self._make_url_result(video_id)

From f5c2c062310b347cd2737920e3eb15f2aeb633d7 Mon Sep 17 00:00:00 2001
From: schnusch <schnusch@users.noreply.github.com>
Date: Sun, 25 Apr 2021 19:32:47 +0200
Subject: [PATCH 356/860] [xfileshare] Add support for wolfstream.tv (#28858)

---
 youtube_dl/extractor/xfileshare.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index cbd5d1cbb..df9efa9fa 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -58,6 +58,7 @@ class XFileShareIE(InfoExtractor):
         (r'vidlocker\.xyz', 'VidLocker'),
         (r'vidshare\.tv', 'VidShare'),
         (r'vup\.to', 'VUp'),
+        (r'wolfstream\.tv', 'WolfStream'),
         (r'xvideosharing\.com', 'XVideoSharing'),
     )
 
@@ -82,6 +83,9 @@ class XFileShareIE(InfoExtractor):
     }, {
         'url': 'https://aparat.cam/n4d6dh0wvlpr',
         'only_matching': True,
+    }, {
+        'url': 'https://wolfstream.tv/nthme29v9u2x',
+        'only_matching': True,
     }]
 
     @staticmethod

From 346dd3b5e87503c52fc6800d9f73cd6bdbce71bc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Apr 2021 01:29:50 +0700
Subject: [PATCH 357/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 45d5c2ebf..c59984d63 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,16 @@
+version <unreleased>
+
+Extractors
++ [xfileshare] Add support for wolfstream.tv (#28858)
+* [francetvinfo] Improve video id extraction (#28792)
+* [medaltv] Fix extraction (#28807)
+* [tver] Redirect all downloads to Brightcove (#28849)
+* [go] Improve video id extraction (#25207, #25216, #26058)
+* [youtube] Fix lazy extractors (#28780)
++ [bbc] Extract description and timestamp from __INITIAL_DATA__ (#28774)
+* [cbsnews] Fix extraction for python <3.6 (#23359)
+
+
 version 2021.04.17
 
 Core

From 273964d190fb048477e71114c4734fcb819c5c16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Apr 2021 01:33:30 +0700
Subject: [PATCH 358/860] release 2021.04.26

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 docs/supportedsites.md                           | 2 +-
 youtube_dl/version.py                            | 2 +-
 8 files changed, 15 insertions(+), 15 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index e2a89d5c2..6ece3e031 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.04.17**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.26**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.04.17
+ [debug] youtube-dl version 2021.04.26
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index 4d7abd775..f923b2d5f 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.04.17**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.26**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index d8dce6fd4..97d605653 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.04.17**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.26**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index d95ee291a..73a806833 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.04.17**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.26**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.04.17
+ [debug] youtube-dl version 2021.04.26
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index ac5dd2f27..ee19a75f5 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.04.17**
+- [ ] I've verified that I'm running youtube-dl version **2021.04.26**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index c59984d63..f15c84225 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.04.26
 
 Extractors
 + [xfileshare] Add support for wolfstream.tv (#28858)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index a23da1a31..88d474de4 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -1162,7 +1162,7 @@
  - **WWE**
  - **XBef**
  - **XboxClips**
- - **XFileShare**: XFileShare based sites: Aparat, ClipWatching, GoUnlimited, GoVid, HolaVid, Streamty, TheVideoBee, Uqload, VidBom, vidlo, VidLocker, VidShare, VUp, XVideoSharing
+ - **XFileShare**: XFileShare based sites: Aparat, ClipWatching, GoUnlimited, GoVid, HolaVid, Streamty, TheVideoBee, Uqload, VidBom, vidlo, VidLocker, VidShare, VUp, WolfStream, XVideoSharing
  - **XHamster**
  - **XHamsterEmbed**
  - **XHamsterUser**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 2b041d593..576f721db 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.04.17'
+__version__ = '2021.04.26'

From 94520568b399d5b4f35a9708f5643d8b16c6c4ee Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 26 Apr 2021 02:16:47 +0700
Subject: [PATCH 359/860] [workflows/ci.yml] Update link to jython-installer

---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 97b50aedc..90bd63c32 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -49,7 +49,7 @@ jobs:
     - name: Install Jython
       if: ${{ matrix.python-impl == 'jython' }}
       run: |
-        wget http://search.maven.org/remotecontent?filepath=org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar -O jython-installer.jar
+        wget https://repo1.maven.org/maven2/org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar -O jython-installer.jar
         java -jar jython-installer.jar -s -d "$HOME/jython"
         echo "$HOME/jython/bin" >> $GITHUB_PATH
     - name: Install nose

From e33dfb445c547f210a7060e8b7abd592dbe42808 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 May 2021 17:53:27 +0700
Subject: [PATCH 360/860] [tv2dk] Fix extraction (closes #28888)

---
 youtube_dl/extractor/tv2dk.py | 17 ++++++++++++++---
 1 file changed, 14 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tv2dk.py b/youtube_dl/extractor/tv2dk.py
index 8bda9348d..8bd5fd640 100644
--- a/youtube_dl/extractor/tv2dk.py
+++ b/youtube_dl/extractor/tv2dk.py
@@ -74,6 +74,12 @@ class TV2DKIE(InfoExtractor):
         webpage = self._download_webpage(url, video_id)
 
         entries = []
+
+        def add_entry(partner_id, kaltura_id):
+            entries.append(self.url_result(
+                'kaltura:%s:%s' % (partner_id, kaltura_id), 'Kaltura',
+                video_id=kaltura_id))
+
         for video_el in re.findall(r'(?s)<[^>]+\bdata-entryid\s*=[^>]*>', webpage):
             video = extract_attributes(video_el)
             kaltura_id = video.get('data-entryid')
@@ -82,9 +88,14 @@ class TV2DKIE(InfoExtractor):
             partner_id = video.get('data-partnerid')
             if not partner_id:
                 continue
-            entries.append(self.url_result(
-                'kaltura:%s:%s' % (partner_id, kaltura_id), 'Kaltura',
-                video_id=kaltura_id))
+            add_entry(partner_id, kaltura_id)
+        if not entries:
+            kaltura_id = self._search_regex(
+                r'entry_id\s*:\s*["\']([0-9a-z_]+)', webpage, 'kaltura id')
+            partner_id = self._search_regex(
+                (r'\\u002Fp\\u002F(\d+)\\u002F', r'/p/(\d+)/'), webpage,
+                'partner id')
+            add_entry(partner_id, kaltura_id)
         return self.playlist_result(entries)
 
 

From d2f72c40db0d1fe1102c98c017682b283579ad97 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 May 2021 18:09:32 +0700
Subject: [PATCH 361/860] [svtplay] Improve extraction (closes #28507, closes
 #28876)

---
 youtube_dl/extractor/svt.py | 7 +++++--
 1 file changed, 5 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/svt.py b/youtube_dl/extractor/svt.py
index aba9bb447..a5bb6daa7 100644
--- a/youtube_dl/extractor/svt.py
+++ b/youtube_dl/extractor/svt.py
@@ -146,7 +146,7 @@ class SVTPlayIE(SVTPlayBaseIE):
                         )
                         (?P<svt_id>[^/?#&]+)|
                         https?://(?:www\.)?(?:svtplay|oppetarkiv)\.se/(?:video|klipp|kanaler)/(?P<id>[^/?#&]+)
-                        (?:.*?modalId=(?P<modal_id>[\da-zA-Z-]+))?
+                        (?:.*?(?:modalId|id)=(?P<modal_id>[\da-zA-Z-]+))?
                     )
                     '''
     _TESTS = [{
@@ -177,6 +177,9 @@ class SVTPlayIE(SVTPlayBaseIE):
     }, {
         'url': 'https://www.svtplay.se/video/30479064/husdrommar/husdrommar-sasong-8-designdrommar-i-stenungsund?modalId=8zVbDPA',
         'only_matching': True,
+    }, {
+        'url': 'https://www.svtplay.se/video/30684086/rapport/rapport-24-apr-18-00-7?id=e72gVpa',
+        'only_matching': True,
     }, {
         # geo restricted to Sweden
         'url': 'http://www.oppetarkiv.se/video/5219710/trollflojten',
@@ -259,7 +262,7 @@ class SVTPlayIE(SVTPlayBaseIE):
         if not svt_id:
             svt_id = self._search_regex(
                 (r'<video[^>]+data-video-id=["\']([\da-zA-Z-]+)',
-                 r'<[^>]+\bdata-rt=["\']top-area-play-button["\'][^>]+\bhref=["\'][^"\']*video/%s/[^"\']*\bmodalId=([\da-zA-Z-]+)' % re.escape(video_id),
+                 r'<[^>]+\bdata-rt=["\']top-area-play-button["\'][^>]+\bhref=["\'][^"\']*video/%s/[^"\']*\b(?:modalId|id)=([\da-zA-Z-]+)' % re.escape(video_id),
                  r'["\']videoSvtId["\']\s*:\s*["\']([\da-zA-Z-]+)',
                  r'["\']videoSvtId\\?["\']\s*:\s*\\?["\']([\da-zA-Z-]+)',
                  r'"content"\s*:\s*{.*?"id"\s*:\s*"([\da-zA-Z-]+)"',

From ff04d43c469e4cf8c14ba3e2e79da0d35ef3c7db Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 May 2021 18:33:05 +0700
Subject: [PATCH 362/860] [xtube] Fix formats extraction (closes #28870)

---
 youtube_dl/extractor/xtube.py | 51 ++++++++++++++++++++++++++++-------
 1 file changed, 42 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/xtube.py b/youtube_dl/extractor/xtube.py
index 18969058f..7246409e3 100644
--- a/youtube_dl/extractor/xtube.py
+++ b/youtube_dl/extractor/xtube.py
@@ -11,6 +11,7 @@ from ..utils import (
     parse_duration,
     sanitized_Request,
     str_to_int,
+    url_or_none,
 )
 
 
@@ -87,10 +88,10 @@ class XTubeIE(InfoExtractor):
                 'Cookie': 'age_verified=1; cookiesAccepted=1',
             })
 
-        title, thumbnail, duration = [None] * 3
+        title, thumbnail, duration, sources, media_definition = [None] * 5
 
         config = self._parse_json(self._search_regex(
-            r'playerConf\s*=\s*({.+?})\s*,\s*(?:\n|loaderConf)', webpage, 'config',
+            r'playerConf\s*=\s*({.+?})\s*,\s*(?:\n|loaderConf|playerWrapper)', webpage, 'config',
             default='{}'), video_id, transform_source=js_to_json, fatal=False)
         if config:
             config = config.get('mainRoll')
@@ -99,20 +100,52 @@ class XTubeIE(InfoExtractor):
                 thumbnail = config.get('poster')
                 duration = int_or_none(config.get('duration'))
                 sources = config.get('sources') or config.get('format')
+                media_definition = config.get('mediaDefinition')
 
-        if not isinstance(sources, dict):
+        if not isinstance(sources, dict) and not media_definition:
             sources = self._parse_json(self._search_regex(
                 r'(["\'])?sources\1?\s*:\s*(?P<sources>{.+?}),',
                 webpage, 'sources', group='sources'), video_id,
                 transform_source=js_to_json)
 
         formats = []
-        for format_id, format_url in sources.items():
-            formats.append({
-                'url': format_url,
-                'format_id': format_id,
-                'height': int_or_none(format_id),
-            })
+        format_urls = set()
+
+        if isinstance(sources, dict):
+            for format_id, format_url in sources.items():
+                format_url = url_or_none(format_url)
+                if not format_url:
+                    continue
+                if format_url in format_urls:
+                    continue
+                format_urls.add(format_url)
+                formats.append({
+                    'url': format_url,
+                    'format_id': format_id,
+                    'height': int_or_none(format_id),
+                })
+
+        if isinstance(media_definition, list):
+            for media in media_definition:
+                video_url = url_or_none(media.get('videoUrl'))
+                if not video_url:
+                    continue
+                if video_url in format_urls:
+                    continue
+                format_urls.add(video_url)
+                format_id = media.get('format')
+                if format_id == 'hls':
+                    formats.extend(self._extract_m3u8_formats(
+                        video_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                        m3u8_id='hls', fatal=False))
+                elif format_id == 'mp4':
+                    height = int_or_none(media.get('quality'))
+                    formats.append({
+                        'url': video_url,
+                        'format_id': '%s-%d' % (format_id, height) if height else format_id,
+                        'height': height,
+                    })
+
         self._remove_duplicate_formats(formats)
         self._sort_formats(formats)
 

From d1b9a5e2eff1c075b38815a3d2b25eb8b3f626bb Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 May 2021 19:00:39 +0700
Subject: [PATCH 363/860] [twitter] Improve formats extraction from vmap URL
 (closes #28909)

---
 youtube_dl/extractor/twitter.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/twitter.py b/youtube_dl/extractor/twitter.py
index ed495f297..cfa7a7326 100644
--- a/youtube_dl/extractor/twitter.py
+++ b/youtube_dl/extractor/twitter.py
@@ -19,6 +19,7 @@ from ..utils import (
     strip_or_none,
     unified_timestamp,
     update_url_query,
+    url_or_none,
     xpath_text,
 )
 
@@ -52,6 +53,9 @@ class TwitterBaseIE(InfoExtractor):
             return [f]
 
     def _extract_formats_from_vmap_url(self, vmap_url, video_id):
+        vmap_url = url_or_none(vmap_url)
+        if not vmap_url:
+            return []
         vmap_data = self._download_xml(vmap_url, video_id)
         formats = []
         urls = []

From a0df8a06178e530a1097f177a1faf1d2c609ac99 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 1 May 2021 22:53:30 +0700
Subject: [PATCH 364/860] [cda] Improve extraction (closes #28709, closes
 #28937)

---
 youtube_dl/extractor/cda.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/cda.py b/youtube_dl/extractor/cda.py
index 1b4362144..e1b391937 100644
--- a/youtube_dl/extractor/cda.py
+++ b/youtube_dl/extractor/cda.py
@@ -133,6 +133,8 @@ class CDAIE(InfoExtractor):
             'age_limit': 18 if need_confirm_age else 0,
         }
 
+        info = self._search_json_ld(webpage, video_id, default={})
+
         # Source: https://www.cda.pl/js/player.js?t=1606154898
         def decrypt_file(a):
             for p in ('_XDDD', '_CDA', '_ADC', '_CXD', '_QWE', '_Q5', '_IKSDE'):
@@ -197,7 +199,7 @@ class CDAIE(InfoExtractor):
                 handler = self._download_webpage
 
             webpage = handler(
-                self._BASE_URL + href, video_id,
+                urljoin(self._BASE_URL, href), video_id,
                 'Downloading %s version information' % resolution, fatal=False)
             if not webpage:
                 # Manually report warning because empty page is returned when
@@ -209,6 +211,4 @@ class CDAIE(InfoExtractor):
 
         self._sort_formats(formats)
 
-        info = self._search_json_ld(webpage, video_id, default={})
-
         return merge_dicts(info_dict, info)

From 0204838163bd4068fe23b40414573d1307d817ab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 May 2021 01:57:02 +0700
Subject: [PATCH 365/860] [kaltura] Make embed code alternatives actually work

---
 youtube_dl/extractor/kaltura.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 49d13460d..5d0ff0418 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -120,7 +120,7 @@ class KalturaIE(InfoExtractor):
     def _extract_urls(webpage):
         # Embed codes: https://knowledge.kaltura.com/embedding-kaltura-media-players-your-site
         finditer = (
-            re.finditer(
+            list(re.finditer(
                 r"""(?xs)
                     kWidget\.(?:thumb)?[Ee]mbed\(
                     \{.*?
@@ -128,8 +128,8 @@ class KalturaIE(InfoExtractor):
                         (?P<q2>['"])_?(?P<partner_id>(?:(?!(?P=q2)).)+)(?P=q2),.*?
                         (?P<q3>['"])entry_?[Ii]d(?P=q3)\s*:\s*
                         (?P<q4>['"])(?P<id>(?:(?!(?P=q4)).)+)(?P=q4)(?:,|\s*\})
-                """, webpage)
-            or re.finditer(
+                """, webpage))
+            or list(re.finditer(
                 r'''(?xs)
                     (?P<q1>["'])
                         (?:https?:)?//cdnapi(?:sec)?\.kaltura\.com(?::\d+)?/(?:(?!(?P=q1)).)*\b(?:p|partner_id)/(?P<partner_id>\d+)(?:(?!(?P=q1)).)*
@@ -142,8 +142,8 @@ class KalturaIE(InfoExtractor):
                         \[\s*(?P<q2_1>["'])entry_?[Ii]d(?P=q2_1)\s*\]\s*=\s*
                     )
                     (?P<q3>["'])(?P<id>(?:(?!(?P=q3)).)+)(?P=q3)
-                ''', webpage)
-            or re.finditer(
+                ''', webpage))
+            or list(re.finditer(
                 r'''(?xs)
                     <(?:iframe[^>]+src|meta[^>]+\bcontent)=(?P<q1>["'])
                       (?:https?:)?//(?:(?:www|cdnapi(?:sec)?)\.)?kaltura\.com/(?:(?!(?P=q1)).)*\b(?:p|partner_id)/(?P<partner_id>\d+)
@@ -151,7 +151,7 @@ class KalturaIE(InfoExtractor):
                       [?&;]entry_id=(?P<id>(?:(?!(?P=q1))[^&])+)
                       (?:(?!(?P=q1)).)*
                     (?P=q1)
-                ''', webpage)
+                ''', webpage))
         )
         urls = []
         for mobj in finditer:

From fe05191b8c59538a48b6cbc95f4fe54fc7e6a0ac Mon Sep 17 00:00:00 2001
From: Ben Rog-Wilhelm <zorba-github@pavlovian.net>
Date: Tue, 4 May 2021 14:14:35 -0500
Subject: [PATCH 366/860] [kaltura] Improve iframe extraction (#28969)

Co-authored-by: Sergey M. <dstftw@gmail.com>
---
 youtube_dl/extractor/gdcvault.py | 15 +++++++++++++++
 youtube_dl/extractor/kaltura.py  |  2 +-
 2 files changed, 16 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 2f555c1d4..5ad40ee23 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -102,6 +102,21 @@ class GDCVaultIE(InfoExtractor):
                 'format': 'mp4-408',
             },
         },
+        {
+            # Kaltura embed, whitespace between quote and embedded URL in iframe's src
+            'url': 'https://www.gdcvault.com/play/1025699',
+            'info_dict': {
+                'id': '0_zagynv0a',
+                'ext': 'mp4',
+                'title': 'Tech Toolbox',
+                'upload_date': '20190408',
+                'uploader_id': 'joe@blazestreaming.com',
+                'timestamp': 1554764629,
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
     ]
 
     def _login(self, webpage_url, display_id):
diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index 5d0ff0418..c731612c4 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -145,7 +145,7 @@ class KalturaIE(InfoExtractor):
                 ''', webpage))
             or list(re.finditer(
                 r'''(?xs)
-                    <(?:iframe[^>]+src|meta[^>]+\bcontent)=(?P<q1>["'])
+                    <(?:iframe[^>]+src|meta[^>]+\bcontent)=(?P<q1>["'])\s*
                       (?:https?:)?//(?:(?:www|cdnapi(?:sec)?)\.)?kaltura\.com/(?:(?!(?P=q1)).)*\b(?:p|partner_id)/(?P<partner_id>\d+)
                       (?:(?!(?P=q1)).)*
                       [?&;]entry_id=(?P<id>(?:(?!(?P=q1))[^&])+)

From b8645c1f5885522ec8bb77649f49ce842e947c25 Mon Sep 17 00:00:00 2001
From: Ben Rog-Wilhelm <zorba-github@pavlovian.net>
Date: Sat, 17 Apr 2021 23:15:10 -0500
Subject: [PATCH 367/860] [dispeak] Improve FLV extraction (closes #13513)

---
 youtube_dl/extractor/dispeak.py | 50 ++++++++++++++++++++-------------
 1 file changed, 30 insertions(+), 20 deletions(-)

diff --git a/youtube_dl/extractor/dispeak.py b/youtube_dl/extractor/dispeak.py
index c345e0274..e776ac00c 100644
--- a/youtube_dl/extractor/dispeak.py
+++ b/youtube_dl/extractor/dispeak.py
@@ -32,6 +32,14 @@ class DigitallySpeakingIE(InfoExtractor):
         # From http://www.gdcvault.com/play/1013700/Advanced-Material
         'url': 'http://sevt.dispeak.com/ubm/gdc/eur10/xml/11256_1282118587281VNIT.xml',
         'only_matching': True,
+    }, {
+        # From https://gdcvault.com/play/1016624
+        'url': 'https://sevt.dispeak.com/ubm/gdc/online12/xml/201210-822101_1349794556671DDDD.xml',
+        'info_dict': {
+            'id': '201210-822101_1349794556671DDDD',
+            'ext': 'flv',
+            'title': 'Pre-launch - Preparing to Take the Plunge',
+        },
     }]
 
     def _parse_mp4(self, metadata):
@@ -84,26 +92,28 @@ class DigitallySpeakingIE(InfoExtractor):
                 'vcodec': 'none',
                 'format_id': audio.get('code'),
             })
-        slide_video_path = xpath_text(metadata, './slideVideo', fatal=True)
-        formats.append({
-            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
-            'play_path': remove_end(slide_video_path, '.flv'),
-            'ext': 'flv',
-            'format_note': 'slide deck video',
-            'quality': -2,
-            'preference': -2,
-            'format_id': 'slides',
-        })
-        speaker_video_path = xpath_text(metadata, './speakerVideo', fatal=True)
-        formats.append({
-            'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
-            'play_path': remove_end(speaker_video_path, '.flv'),
-            'ext': 'flv',
-            'format_note': 'speaker video',
-            'quality': -1,
-            'preference': -1,
-            'format_id': 'speaker',
-        })
+        slide_video_path = xpath_text(metadata, './slideVideo')
+        if slide_video_path:
+            formats.append({
+                'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
+                'play_path': remove_end(slide_video_path, '.flv'),
+                'ext': 'flv',
+                'format_note': 'slide deck video',
+                'quality': -2,
+                'preference': -2,
+                'format_id': 'slides',
+            })
+        speaker_video_path = xpath_text(metadata, './speakerVideo')
+        if speaker_video_path:
+            formats.append({
+                'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
+                'play_path': remove_end(speaker_video_path, '.flv'),
+                'ext': 'flv',
+                'format_note': 'speaker video',
+                'quality': -1,
+                'preference': -1,
+                'format_id': 'speaker',
+            })
         return formats
 
     def _real_extract(self, url):

From 1786cd3fe4e555b83bdd3eea77ade3477293330d Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 May 2021 02:30:42 +0700
Subject: [PATCH 368/860] [dispeak] DRY and update tests (closes #28970)

---
 youtube_dl/extractor/dispeak.py | 34 +++++++++++++++------------------
 1 file changed, 15 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/dispeak.py b/youtube_dl/extractor/dispeak.py
index e776ac00c..276fd4b09 100644
--- a/youtube_dl/extractor/dispeak.py
+++ b/youtube_dl/extractor/dispeak.py
@@ -33,13 +33,17 @@ class DigitallySpeakingIE(InfoExtractor):
         'url': 'http://sevt.dispeak.com/ubm/gdc/eur10/xml/11256_1282118587281VNIT.xml',
         'only_matching': True,
     }, {
-        # From https://gdcvault.com/play/1016624
+        # From https://gdcvault.com/play/1016624, empty speakerVideo
         'url': 'https://sevt.dispeak.com/ubm/gdc/online12/xml/201210-822101_1349794556671DDDD.xml',
         'info_dict': {
             'id': '201210-822101_1349794556671DDDD',
             'ext': 'flv',
             'title': 'Pre-launch - Preparing to Take the Plunge',
         },
+    }, {
+        # From http://www.gdcvault.com/play/1014846/Conference-Keynote-Shigeru, empty slideVideo
+        'url': 'http://events.digitallyspeaking.com/gdc/project25/xml/p25-miyamoto1999_1282467389849HSVB.xml',
+        'only_matching': True,
     }]
 
     def _parse_mp4(self, metadata):
@@ -92,27 +96,19 @@ class DigitallySpeakingIE(InfoExtractor):
                 'vcodec': 'none',
                 'format_id': audio.get('code'),
             })
-        slide_video_path = xpath_text(metadata, './slideVideo')
-        if slide_video_path:
+        for video_key, format_id, preference in (
+                ('slide', 'slides', -2), ('speaker', 'speaker', -1)):
+            video_path = xpath_text(metadata, './%sVideo' % video_key)
+            if not video_path:
+                continue
             formats.append({
                 'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
-                'play_path': remove_end(slide_video_path, '.flv'),
+                'play_path': remove_end(video_path, '.flv'),
                 'ext': 'flv',
-                'format_note': 'slide deck video',
-                'quality': -2,
-                'preference': -2,
-                'format_id': 'slides',
-            })
-        speaker_video_path = xpath_text(metadata, './speakerVideo')
-        if speaker_video_path:
-            formats.append({
-                'url': 'rtmp://%s/ondemand?ovpfv=1.1' % akamai_url,
-                'play_path': remove_end(speaker_video_path, '.flv'),
-                'ext': 'flv',
-                'format_note': 'speaker video',
-                'quality': -1,
-                'preference': -1,
-                'format_id': 'speaker',
+                'format_note': '%s video' % video_key,
+                'quality': preference,
+                'preference': preference,
+                'format_id': format_id,
             })
         return formats
 

From 504e4d804df0ee666d80ba6796017cf97e026c0b Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 May 2021 02:44:29 +0700
Subject: [PATCH 369/860] [gdcvault] Add support for HTML5 videos

---
 youtube_dl/extractor/gdcvault.py | 19 ++++++++++++++++++-
 1 file changed, 18 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/gdcvault.py b/youtube_dl/extractor/gdcvault.py
index 5ad40ee23..acc6478b8 100644
--- a/youtube_dl/extractor/gdcvault.py
+++ b/youtube_dl/extractor/gdcvault.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from .kaltura import KalturaIE
 from ..utils import (
     HEADRequest,
+    remove_start,
     sanitized_Request,
     smuggle_url,
     urlencode_postdata,
@@ -117,6 +118,11 @@ class GDCVaultIE(InfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # HTML5 video
+            'url': 'http://www.gdcvault.com/play/1014846/Conference-Keynote-Shigeru',
+            'only_matching': True,
+        },
     ]
 
     def _login(self, webpage_url, display_id):
@@ -190,7 +196,18 @@ class GDCVaultIE(InfoExtractor):
 
             xml_name = self._html_search_regex(
                 r'<iframe src=".*?\?xml(?:=|URL=xml/)(.+?\.xml).*?".*?</iframe>',
-                start_page, 'xml filename')
+                start_page, 'xml filename', default=None)
+            if not xml_name:
+                info = self._parse_html5_media_entries(url, start_page, video_id)[0]
+                info.update({
+                    'title': remove_start(self._search_regex(
+                        r'>Session Name:\s*<.*?>\s*<td>(.+?)</td>', start_page,
+                        'title', default=None) or self._og_search_title(
+                        start_page, default=None), 'GDC Vault - '),
+                    'id': video_id,
+                    'display_id': display_id,
+                })
+                return info
             embed_url = '%s/xml/%s' % (xml_root, xml_name)
             ie_key = 'DigitallySpeaking'
 

From 04be55307a3a13f7091a9a2970a1cc20863769e3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 May 2021 02:54:12 +0700
Subject: [PATCH 370/860] [funimation] Add support for optional lang code in
 URLs (closes #28950)

---
 youtube_dl/extractor/funimation.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/funimation.py b/youtube_dl/extractor/funimation.py
index 8bbedca26..d8f1e169a 100644
--- a/youtube_dl/extractor/funimation.py
+++ b/youtube_dl/extractor/funimation.py
@@ -16,7 +16,7 @@ from ..utils import (
 
 
 class FunimationIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?funimation(?:\.com|now\.uk)/shows/[^/]+/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https?://(?:www\.)?funimation(?:\.com|now\.uk)/(?:[^/]+/)?shows/[^/]+/(?P<id>[^/?#&]+)'
 
     _NETRC_MACHINE = 'funimation'
     _TOKEN = None
@@ -51,6 +51,10 @@ class FunimationIE(InfoExtractor):
     }, {
         'url': 'https://www.funimationnow.uk/shows/puzzle-dragons-x/drop-impact/simulcast/',
         'only_matching': True,
+    }, {
+        # with lang code
+        'url': 'https://www.funimation.com/en/shows/hacksign/role-play/',
+        'only_matching': True,
     }]
 
     def _login(self):

From b797c1cc750b1f617678281731303611e21c70d0 Mon Sep 17 00:00:00 2001
From: Jacob Chapman <7908073+chapmanjacobd@users.noreply.github.com>
Date: Wed, 5 May 2021 03:31:24 +0700
Subject: [PATCH 371/860] [YoutubeDL] Improve extract_info doc (#28946)

Co-authored-by: Sergey M. <dstftw@gmail.com>
---
 youtube_dl/YoutubeDL.py | 19 ++++++++++++++-----
 1 file changed, 14 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8f65c6499..fe30758ef 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -773,11 +773,20 @@ class YoutubeDL(object):
 
     def extract_info(self, url, download=True, ie_key=None, extra_info={},
                      process=True, force_generic_extractor=False):
-        '''
-        Returns a list with a dictionary for each video we find.
-        If 'download', also downloads the videos.
-        extra_info is a dict containing the extra values to add to each result
-        '''
+        """
+        Return a list with a dictionary for each video extracted.
+
+        Arguments:
+        url -- URL to extract
+
+        Keyword arguments:
+        download -- whether to download videos during extraction
+        ie_key -- extractor key hint
+        extra_info -- dictionary containing the extra values to add to each result
+        process -- whether to resolve all unresolved references (URLs, playlist items),
+            must be True for download to work.
+        force_generic_extractor -- force using the generic extractor
+        """
 
         if not ie_key and force_generic_extractor:
             ie_key = 'Generic'

From 03afef753878c2528260dc8084092556428f4343 Mon Sep 17 00:00:00 2001
From: catboy <79282513+catboy-oss@users.noreply.github.com>
Date: Tue, 4 May 2021 20:44:07 +0000
Subject: [PATCH 372/860] [medaltv] Relax _VALID_URL (#28884)

Co-authored-by: Sergey M. <dstftw@gmail.com>
---
 youtube_dl/extractor/medaltv.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/medaltv.py b/youtube_dl/extractor/medaltv.py
index ef2283dea..67bb4debb 100644
--- a/youtube_dl/extractor/medaltv.py
+++ b/youtube_dl/extractor/medaltv.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class MedalTVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?medal\.tv/clips/(?P<id>[a-zA-Z0-9]+)'
+    _VALID_URL = r'https?://(?:www\.)?medal\.tv/clips/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://medal.tv/clips/2mA60jWAGQCBH',
         'md5': '7b07b064331b1cf9e8e5c52a06ae68fa',
@@ -42,6 +42,12 @@ class MedalTVIE(InfoExtractor):
             'upload_date': '20201117',
             'uploader_id': '5156321',
         }
+    }, {
+        'url': 'https://medal.tv/clips/37rMeFpryCC-9',
+        'only_matching': True,
+    }, {
+        'url': 'https://medal.tv/clips/2WRj40tpY_EU9',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From a7260099873acc6dc7d76cafad2f6b139087afd0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Wed, 5 May 2021 04:12:35 +0700
Subject: [PATCH 373/860] [blinkx] Remove extractor (closes #28941)

No longer exists.
---
 youtube_dl/extractor/blinkx.py     | 86 ------------------------------
 youtube_dl/extractor/extractors.py |  1 -
 2 files changed, 87 deletions(-)
 delete mode 100644 youtube_dl/extractor/blinkx.py

diff --git a/youtube_dl/extractor/blinkx.py b/youtube_dl/extractor/blinkx.py
deleted file mode 100644
index db5e12b21..000000000
--- a/youtube_dl/extractor/blinkx.py
+++ /dev/null
@@ -1,86 +0,0 @@
-from __future__ import unicode_literals
-
-import json
-
-from .common import InfoExtractor
-from ..utils import (
-    remove_start,
-    int_or_none,
-)
-
-
-class BlinkxIE(InfoExtractor):
-    _VALID_URL = r'(?:https?://(?:www\.)blinkx\.com/#?ce/|blinkx:)(?P<id>[^?]+)'
-    IE_NAME = 'blinkx'
-
-    _TEST = {
-        'url': 'http://www.blinkx.com/ce/Da0Gw3xc5ucpNduzLuDDlv4WC9PuI4fDi1-t6Y3LyfdY2SZS5Urbvn-UPJvrvbo8LTKTc67Wu2rPKSQDJyZeeORCR8bYkhs8lI7eqddznH2ofh5WEEdjYXnoRtj7ByQwt7atMErmXIeYKPsSDuMAAqJDlQZ-3Ff4HJVeH_s3Gh8oQ',
-        'md5': '337cf7a344663ec79bf93a526a2e06c7',
-        'info_dict': {
-            'id': 'Da0Gw3xc',
-            'ext': 'mp4',
-            'title': 'No Daily Show for John Oliver; HBO Show Renewed - IGN News',
-            'uploader': 'IGN News',
-            'upload_date': '20150217',
-            'timestamp': 1424215740,
-            'description': 'HBO has renewed Last Week Tonight With John Oliver for two more seasons.',
-            'duration': 47.743333,
-        },
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        display_id = video_id[:8]
-
-        api_url = ('https://apib4.blinkx.com/api.php?action=play_video&'
-                   + 'video=%s' % video_id)
-        data_json = self._download_webpage(api_url, display_id)
-        data = json.loads(data_json)['api']['results'][0]
-        duration = None
-        thumbnails = []
-        formats = []
-        for m in data['media']:
-            if m['type'] == 'jpg':
-                thumbnails.append({
-                    'url': m['link'],
-                    'width': int(m['w']),
-                    'height': int(m['h']),
-                })
-            elif m['type'] == 'original':
-                duration = float(m['d'])
-            elif m['type'] == 'youtube':
-                yt_id = m['link']
-                self.to_screen('Youtube video detected: %s' % yt_id)
-                return self.url_result(yt_id, 'Youtube', video_id=yt_id)
-            elif m['type'] in ('flv', 'mp4'):
-                vcodec = remove_start(m['vcodec'], 'ff')
-                acodec = remove_start(m['acodec'], 'ff')
-                vbr = int_or_none(m.get('vbr') or m.get('vbitrate'), 1000)
-                abr = int_or_none(m.get('abr') or m.get('abitrate'), 1000)
-                tbr = vbr + abr if vbr and abr else None
-                format_id = '%s-%sk-%s' % (vcodec, tbr, m['w'])
-                formats.append({
-                    'format_id': format_id,
-                    'url': m['link'],
-                    'vcodec': vcodec,
-                    'acodec': acodec,
-                    'abr': abr,
-                    'vbr': vbr,
-                    'tbr': tbr,
-                    'width': int_or_none(m.get('w')),
-                    'height': int_or_none(m.get('h')),
-                })
-
-        self._sort_formats(formats)
-
-        return {
-            'id': display_id,
-            'fullid': video_id,
-            'title': data['title'],
-            'formats': formats,
-            'uploader': data['channel_name'],
-            'timestamp': data['pubdate_epoch'],
-            'description': data.get('description'),
-            'thumbnails': thumbnails,
-            'duration': duration,
-        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index ac33cd996..71584b1e6 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -132,7 +132,6 @@ from .bleacherreport import (
     BleacherReportIE,
     BleacherReportCMSIE,
 )
-from .blinkx import BlinkxIE
 from .bloomberg import BloombergIE
 from .bokecc import BokeCCIE
 from .bongacams import BongaCamsIE

From 2202cef0e4551293913e0be06b72b4cffccae0aa Mon Sep 17 00:00:00 2001
From: Lukas Anzinger <lukas@lukasanzinger.at>
Date: Sun, 16 May 2021 14:54:15 +0200
Subject: [PATCH 374/860] [orf:radio] Switch download URLs to HTTPS (closes
 #29012) (#29046)

---
 youtube_dl/extractor/orf.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 700ce448c..3fadbcbea 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -182,7 +182,7 @@ class ORFRadioIE(InfoExtractor):
             duration = end - start if end and start else None
             entries.append({
                 'id': loop_stream_id.replace('.mp3', ''),
-                'url': 'http://loopstream01.apa.at/?channel=%s&id=%s' % (self._LOOP_STATION, loop_stream_id),
+                'url': 'https://loopstream01.apa.at/?channel=%s&id=%s' % (self._LOOP_STATION, loop_stream_id),
                 'title': title,
                 'description': clean_html(data.get('subtitle')),
                 'duration': duration,

From 552b1399110360232bf1bf2ba5ed50cb8b30e818 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 20:28:32 +0700
Subject: [PATCH 375/860] [generic] Add Referer header for direct videojs
 download URLs (closes #2879, closes #20217, closes #29053)

---
 youtube_dl/extractor/generic.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index f99d887ca..648a58c77 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -3400,6 +3400,9 @@ class GenericIE(InfoExtractor):
                         'url': src,
                         'ext': (mimetype2ext(src_type)
                                 or ext if ext in KNOWN_EXTENSIONS else 'mp4'),
+                        'http_headers': {
+                            'Referer': full_response.geturl(),
+                        },
                     })
             if formats:
                 self._sort_formats(formats)

From 8536dcafd82380c006a25409bdc2c3dc0d14195c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 20:48:24 +0700
Subject: [PATCH 376/860] [vk] Add support for sibnet embeds (closes #9500)

---
 youtube_dl/extractor/vk.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/vk.py b/youtube_dl/extractor/vk.py
index 00ec006c4..6b3513ee0 100644
--- a/youtube_dl/extractor/vk.py
+++ b/youtube_dl/extractor/vk.py
@@ -300,6 +300,13 @@ class VKIE(VKBaseIE):
             'only_matching': True,
         }]
 
+    @staticmethod
+    def _extract_sibnet_urls(webpage):
+        # https://help.sibnet.ru/?sibnet_video_embed
+        return [unescapeHTML(mobj.group('url')) for mobj in re.finditer(
+            r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//video\.sibnet\.ru/shell\.php\?.*?\bvideoid=\d+.*?)\1',
+            webpage)]
+
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
         video_id = mobj.group('videoid')
@@ -408,6 +415,10 @@ class VKIE(VKBaseIE):
         if odnoklassniki_url:
             return self.url_result(odnoklassniki_url, OdnoklassnikiIE.ie_key())
 
+        sibnet_urls = self._extract_sibnet_urls(info_page)
+        if sibnet_urls:
+            return self.url_result(sibnet_urls[0])
+
         m_opts = re.search(r'(?s)var\s+opts\s*=\s*({.+?});', info_page)
         if m_opts:
             m_opts_url = re.search(r"url\s*:\s*'((?!/\b)[^']+)", m_opts.group(1))

From 286e01ce30b4d4d7a631512c3d1f983b30d9059c Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 20:50:32 +0700
Subject: [PATCH 377/860] [generic] Add support for sibnet embeds

---
 youtube_dl/extractor/generic.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 648a58c77..7b6f07318 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -126,6 +126,7 @@ from .viqeo import ViqeoIE
 from .expressen import ExpressenIE
 from .zype import ZypeIE
 from .odnoklassniki import OdnoklassnikiIE
+from .vk import VKIE
 from .kinja import KinjaEmbedIE
 from .arcpublishing import ArcPublishingIE
 from .medialaan import MedialaanIE
@@ -2248,6 +2249,11 @@ class GenericIE(InfoExtractor):
             },
             'playlist_mincount': 52,
         },
+        {
+            # Sibnet embed (https://help.sibnet.ru/?sibnet_video_embed)
+            'url': 'https://phpbb3.x-tk.ru/bbcode-video-sibnet-t24.html',
+            'only_matching': True,
+        },
     ]
 
     def report_following_redirect(self, new_url):
@@ -2777,6 +2783,11 @@ class GenericIE(InfoExtractor):
         if odnoklassniki_url:
             return self.url_result(odnoklassniki_url, OdnoklassnikiIE.ie_key())
 
+        # Look for sibnet embedded player
+        sibnet_urls = VKIE._extract_sibnet_urls(webpage)
+        if sibnet_urls:
+            return self.playlist_from_matches(sibnet_urls, video_id, video_title)
+
         # Look for embedded ivi player
         mobj = re.search(r'<embed[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?ivi\.ru/video/player.+?)\1', webpage)
         if mobj is not None:

From eb5080286a8882eedbb77d1a8cd72f1c85b75737 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 21:21:14 +0700
Subject: [PATCH 378/860] [phoenix] Fix extraction (closes #29057)

---
 youtube_dl/extractor/phoenix.py | 51 ++++++++++++++++++---------------
 1 file changed, 28 insertions(+), 23 deletions(-)

diff --git a/youtube_dl/extractor/phoenix.py b/youtube_dl/extractor/phoenix.py
index dbbfce983..e3ea01443 100644
--- a/youtube_dl/extractor/phoenix.py
+++ b/youtube_dl/extractor/phoenix.py
@@ -9,8 +9,9 @@ from ..compat import compat_str
 from ..utils import (
     int_or_none,
     merge_dicts,
+    try_get,
     unified_timestamp,
-    xpath_text,
+    urljoin,
 )
 
 
@@ -27,10 +28,11 @@ class PhoenixIE(ZDFBaseIE):
             'title': 'Wohin führt der Protest in der Pandemie?',
             'description': 'md5:7d643fe7f565e53a24aac036b2122fbd',
             'duration': 1691,
-            'timestamp': 1613906100,
+            'timestamp': 1613902500,
             'upload_date': '20210221',
             'uploader': 'Phoenix',
-            'channel': 'corona nachgehakt',
+            'series': 'corona nachgehakt',
+            'episode': 'Wohin führt der Protest in der Pandemie?',
         },
     }, {
         # Youtube embed
@@ -79,50 +81,53 @@ class PhoenixIE(ZDFBaseIE):
 
         video_id = compat_str(video.get('basename') or video.get('content'))
 
-        details = self._download_xml(
+        details = self._download_json(
             'https://www.phoenix.de/php/mediaplayer/data/beitrags_details.php',
-            video_id, 'Downloading details XML', query={
+            video_id, 'Downloading details JSON', query={
                 'ak': 'web',
                 'ptmd': 'true',
                 'id': video_id,
                 'profile': 'player2',
             })
 
-        title = title or xpath_text(
-            details, './/information/title', 'title', fatal=True)
-        content_id = xpath_text(
-            details, './/video/details/basename', 'content id', fatal=True)
+        title = title or details['title']
+        content_id = details['tracking']['nielsen']['content']['assetid']
 
         info = self._extract_ptmd(
             'https://tmd.phoenix.de/tmd/2/ngplayer_2_3/vod/ptmd/phoenix/%s' % content_id,
             content_id, None, url)
 
-        timestamp = unified_timestamp(xpath_text(details, './/details/airtime'))
+        duration = int_or_none(try_get(
+            details, lambda x: x['tracking']['nielsen']['content']['length']))
+        timestamp = unified_timestamp(details.get('editorialDate'))
+        series = try_get(
+            details, lambda x: x['tracking']['nielsen']['content']['program'],
+            compat_str)
+        episode = title if details.get('contentType') == 'episode' else None
 
         thumbnails = []
-        for node in details.findall('.//teaserimages/teaserimage'):
-            thumbnail_url = node.text
+        teaser_images = try_get(details, lambda x: x['teaserImageRef']['layouts'], dict) or {}
+        for thumbnail_key, thumbnail_url in teaser_images.items():
+            thumbnail_url = urljoin(url, thumbnail_url)
             if not thumbnail_url:
                 continue
             thumbnail = {
                 'url': thumbnail_url,
             }
-            thumbnail_key = node.get('key')
-            if thumbnail_key:
-                m = re.match('^([0-9]+)x([0-9]+)$', thumbnail_key)
-                if m:
-                    thumbnail['width'] = int(m.group(1))
-                    thumbnail['height'] = int(m.group(2))
+            m = re.match('^([0-9]+)x([0-9]+)$', thumbnail_key)
+            if m:
+                thumbnail['width'] = int(m.group(1))
+                thumbnail['height'] = int(m.group(2))
             thumbnails.append(thumbnail)
 
         return merge_dicts(info, {
             'id': content_id,
             'title': title,
-            'description': xpath_text(details, './/information/detail'),
-            'duration': int_or_none(xpath_text(details, './/details/lengthSec')),
+            'description': details.get('leadParagraph'),
+            'duration': duration,
             'thumbnails': thumbnails,
             'timestamp': timestamp,
-            'uploader': xpath_text(details, './/details/channel'),
-            'uploader_id': xpath_text(details, './/details/originChannelId'),
-            'channel': xpath_text(details, './/details/originChannelTitle'),
+            'uploader': details.get('tvService'),
+            'series': series,
+            'episode': episode,
         })

From 6423d7054eb2a73a1557c8531f631330e6d7e084 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 21:34:10 +0700
Subject: [PATCH 379/860] [options] Fix thumbnail option group name (closes
 #29042)

---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 241cf110f..0a0641bd4 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -768,7 +768,7 @@ def parseOpts(overrideArguments=None):
         action='store_true', dest='rm_cachedir',
         help='Delete all filesystem cache files')
 
-    thumbnail = optparse.OptionGroup(parser, 'Thumbnail images')
+    thumbnail = optparse.OptionGroup(parser, 'Thumbnail Options')
     thumbnail.add_option(
         '--write-thumbnail',
         action='store_true', dest='writethumbnail', default=False,

From 1e8aaa1d155d2f6b04ba1caa040876372bb0bb44 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 21:42:38 +0700
Subject: [PATCH 380/860] [generic] Add support for og:audio (closes #28311,
 closes #29015)

---
 youtube_dl/extractor/generic.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 7b6f07318..87594534f 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -3482,7 +3482,7 @@ class GenericIE(InfoExtractor):
             m_video_type = re.findall(r'<meta.*?property="og:video:type".*?content="video/(.*?)"', webpage)
             # We only look in og:video if the MIME type is a video, don't try if it's a Flash player:
             if m_video_type is not None:
-                found = filter_video(re.findall(r'<meta.*?property="og:video".*?content="(.*?)"', webpage))
+                found = filter_video(re.findall(r'<meta.*?property="og:(?:video|audio)".*?content="(.*?)"', webpage))
         if not found:
             REDIRECT_REGEX = r'[0-9]{,2};\s*(?:URL|url)=\'?([^\'"]+)'
             found = re.search(

From ef03721f470606deaddb4a57e51550045a32fb04 Mon Sep 17 00:00:00 2001
From: kr4ssi <44404263+kr4ssi@users.noreply.github.com>
Date: Sun, 16 May 2021 16:46:32 +0200
Subject: [PATCH 381/860] [vivo] Add support for vivo.st (#29009)

Co-authored-by: Sergey M. <dstftw@gmail.com>
---
 youtube_dl/extractor/shared.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/shared.py b/youtube_dl/extractor/shared.py
index 02295d1a4..93ab2a167 100644
--- a/youtube_dl/extractor/shared.py
+++ b/youtube_dl/extractor/shared.py
@@ -86,10 +86,10 @@ class SharedIE(SharedBaseIE):
 
 class VivoIE(SharedBaseIE):
     IE_DESC = 'vivo.sx'
-    _VALID_URL = r'https?://vivo\.sx/(?P<id>[\da-z]{10})'
+    _VALID_URL = r'https?://vivo\.s[xt]/(?P<id>[\da-z]{10})'
     _FILE_NOT_FOUND = '>The file you have requested does not exists or has been removed'
 
-    _TEST = {
+    _TESTS = [{
         'url': 'http://vivo.sx/d7ddda0e78',
         'md5': '15b3af41be0b4fe01f4df075c2678b2c',
         'info_dict': {
@@ -98,7 +98,10 @@ class VivoIE(SharedBaseIE):
             'title': 'Chicken',
             'filesize': 515659,
         },
-    }
+    }, {
+        'url': 'http://vivo.st/d7ddda0e78',
+        'only_matching': True,
+    }]
 
     def _extract_title(self, webpage):
         title = self._html_search_regex(

From 503a3744ad620a2fe21c82cb2c595dc939310a1e Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 21:56:30 +0700
Subject: [PATCH 382/860] [eroprofile] Fix extraction (closes #23200, closes
 #23626, closes #29008)

---
 youtube_dl/extractor/eroprofile.py | 18 +++++++-----------
 1 file changed, 7 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/eroprofile.py b/youtube_dl/extractor/eroprofile.py
index c08643a17..9ee549d77 100644
--- a/youtube_dl/extractor/eroprofile.py
+++ b/youtube_dl/extractor/eroprofile.py
@@ -6,7 +6,7 @@ from .common import InfoExtractor
 from ..compat import compat_urllib_parse_urlencode
 from ..utils import (
     ExtractorError,
-    unescapeHTML
+    merge_dicts,
 )
 
 
@@ -77,19 +77,15 @@ class EroProfileIE(InfoExtractor):
             [r"glbUpdViews\s*\('\d*','(\d+)'", r'p/report/video/(\d+)'],
             webpage, 'video id', default=None)
 
-        video_url = unescapeHTML(self._search_regex(
-            r'<source src="([^"]+)', webpage, 'video url'))
         title = self._html_search_regex(
-            r'Title:</th><td>([^<]+)</td>', webpage, 'title')
-        thumbnail = self._search_regex(
-            r'onclick="showVideoPlayer\(\)"><img src="([^"]+)',
-            webpage, 'thumbnail', fatal=False)
+            (r'Title:</th><td>([^<]+)</td>', r'<h1[^>]*>(.+?)</h1>'),
+            webpage, 'title')
 
-        return {
+        info = self._parse_html5_media_entries(url, webpage, video_id)[0]
+
+        return merge_dicts(info, {
             'id': video_id,
             'display_id': display_id,
-            'url': video_url,
             'title': title,
-            'thumbnail': thumbnail,
             'age_limit': 18,
-        }
+        })

From 199c645bee2052e43ec33cc8d0b0fa0c18853da8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 22:01:51 +0700
Subject: [PATCH 383/860] [eroprofile] Skip test

---
 youtube_dl/extractor/eroprofile.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/eroprofile.py b/youtube_dl/extractor/eroprofile.py
index 9ee549d77..c460dc7f9 100644
--- a/youtube_dl/extractor/eroprofile.py
+++ b/youtube_dl/extractor/eroprofile.py
@@ -24,7 +24,8 @@ class EroProfileIE(InfoExtractor):
             'title': 'sexy babe softcore',
             'thumbnail': r're:https?://.*\.jpg',
             'age_limit': 18,
-        }
+        },
+        'skip': 'Video not found',
     }, {
         'url': 'http://www.eroprofile.com/m/videos/view/Try-It-On-Pee_cut_2-wmv-4shared-com-file-sharing-download-movie-file',
         'md5': '1baa9602ede46ce904c431f5418d8916',

From e90a890f01ad253b611d8edd365f41b0c4553b67 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 22:31:37 +0700
Subject: [PATCH 384/860] [playstuff] Add extractor (closes #28901, closes
 #28931)

---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/playstuff.py  | 65 ++++++++++++++++++++++++++++++
 2 files changed, 66 insertions(+)
 create mode 100644 youtube_dl/extractor/playstuff.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 71584b1e6..402e542ae 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -925,6 +925,7 @@ from .platzi import (
 from .playfm import PlayFMIE
 from .playplustv import PlayPlusTVIE
 from .plays import PlaysTVIE
+from .playstuff import PlayStuffIE
 from .playtvak import PlaytvakIE
 from .playvid import PlayvidIE
 from .playwire import PlaywireIE
diff --git a/youtube_dl/extractor/playstuff.py b/youtube_dl/extractor/playstuff.py
new file mode 100644
index 000000000..5a329957f
--- /dev/null
+++ b/youtube_dl/extractor/playstuff.py
@@ -0,0 +1,65 @@
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    smuggle_url,
+    try_get,
+)
+
+
+class PlayStuffIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?play\.stuff\.co\.nz/details/(?P<id>[^/?#&]+)'
+    _TESTS = [{
+        'url': 'https://play.stuff.co.nz/details/608778ac1de1c4001a3fa09a',
+        'md5': 'c82d3669e5247c64bc382577843e5bd0',
+        'info_dict': {
+            'id': '6250584958001',
+            'ext': 'mp4',
+            'title': 'Episode 1: Rotorua/Mt Maunganui/Tauranga',
+            'description': 'md5:c154bafb9f0dd02d01fd4100fb1c1913',
+            'uploader_id': '6005208634001',
+            'timestamp': 1619491027,
+            'upload_date': '20210427',
+        },
+        'add_ie': ['BrightcoveNew'],
+    }, {
+        # geo restricted, bypassable
+        'url': 'https://play.stuff.co.nz/details/_6155660351001',
+        'only_matching': True,
+    }]
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s'
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+
+        state = self._parse_json(
+            self._search_regex(
+                r'__INITIAL_STATE__\s*=\s*({.+?})\s*;', webpage, 'state'),
+            video_id)
+
+        account_id = try_get(
+            state, lambda x: x['configurations']['accountId'],
+            compat_str) or '6005208634001'
+        player_id = try_get(
+            state, lambda x: x['configurations']['playerId'],
+            compat_str) or 'default'
+
+        entries = []
+        for item_id, video in state['items'].items():
+            if not isinstance(video, dict):
+                continue
+            asset_id = try_get(
+                video, lambda x: x['content']['attributes']['assetId'],
+                compat_str)
+            if not asset_id:
+                continue
+            entries.append(self.url_result(
+                smuggle_url(
+                    self.BRIGHTCOVE_URL_TEMPLATE % (account_id, player_id, asset_id),
+                    {'geo_countries': ['NZ']}),
+                'BrightcoveNew', video_id))
+
+        return self.playlist_result(entries, video_id)

From efeb9e0fbf1e33043c19dbfda9d12984bf0c6e34 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 22:40:39 +0700
Subject: [PATCH 385/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 31 +++++++++++++++++++++++++++++++
 1 file changed, 31 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index f15c84225..7c327ab9d 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,34 @@
+version <unreleased>
+
+Core
+* [options] Fix thumbnail option group name (#29042)
+* [YoutubeDL] Improve extract_info doc (#28946)
+
+Extractors
++ [playstuff] Add support for play.stuff.co.nz (#28901, #28931)
+* [eroprofile] Fix extraction (#23200, #23626, #29008)
++ [vivo] Add support for vivo.st (#29009)
++ [generic] Add support for og:audio (#28311, #29015)
+* [phoenix] Fix extraction (#29057)
++ [generic] Add support for sibnet embeds
++ [vk] Add support for sibnet embeds (#9500)
++ [generic] Add Referer header for direct videojs download URLs (#2879,
+  #20217, #29053)
+* [orf:radio] Switch download URLs to HTTPS (#29012, #29046)
+- [blinkx] Remove extractor (#28941)
+* [medaltv] Relax URL regular expression (#28884)
++ [funimation] Add support for optional lang code in URLs (#28950)
++ [gdcvault] Add support for HTML5 videos
+* [dispeak] Improve FLV extraction (#13513, #28970)
+* [kaltura] Improve iframe extraction (#28969)
+* [kaltura] Make embed code alternatives actually work
+* [cda] Improve extraction (#28709, #28937)
+* [twitter] Improve formats extraction from vmap URL (#28909)
+* [xtube] Fix formats extraction (#28870)
+* [svtplay] Improve extraction (#28507, #28876)
+* [tv2dk] Fix extraction (#28888)
+
+
 version 2021.04.26
 
 Extractors

From f47627a1c9a790fac29d5c166bc0f7944fcb1a98 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 16 May 2021 22:55:05 +0700
Subject: [PATCH 386/860] release 2021.05.16

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 README.md                                        | 2 +-
 docs/supportedsites.md                           | 2 +-
 youtube_dl/version.py                            | 2 +-
 9 files changed, 16 insertions(+), 16 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 6ece3e031..d67bb482c 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.04.26**
+- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.04.26
+ [debug] youtube-dl version 2021.05.16
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index f923b2d5f..efe9fef8c 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.04.26**
+- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 97d605653..e213fc1a9 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.04.26**
+- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index 73a806833..1645087ad 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.04.26**
+- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.04.26
+ [debug] youtube-dl version 2021.05.16
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index ee19a75f5..e6e569af6 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.04.26. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.04.26**
+- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index 7c327ab9d..5ea1d3150 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.05.16
 
 Core
 * [options] Fix thumbnail option group name (#29042)
diff --git a/README.md b/README.md
index 94c34d89a..059141611 100644
--- a/README.md
+++ b/README.md
@@ -287,7 +287,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo
     --no-cache-dir                       Disable filesystem caching
     --rm-cache-dir                       Delete all filesystem cache files
 
-## Thumbnail images:
+## Thumbnail Options:
     --write-thumbnail                    Write thumbnail image to disk
     --write-all-thumbnails               Write all thumbnail image formats to
                                          disk
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index 88d474de4..ed0d5e9d9 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -119,7 +119,6 @@
  - **BitChuteChannel**
  - **BleacherReport**
  - **BleacherReportCMS**
- - **blinkx**
  - **Bloomberg**
  - **BokeCC**
  - **BongaCams**
@@ -713,6 +712,7 @@
  - **play.fm**
  - **player.sky.it**
  - **PlayPlusTV**
+ - **PlayStuff**
  - **PlaysTV**
  - **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz
  - **Playvid**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 576f721db..b16a84100 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.04.26'
+__version__ = '2021.05.16'

From e1a9d0ef780b7a0cdcdc706909ad2de9bd06138c Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 17 May 2021 12:37:39 +0100
Subject: [PATCH 387/860] [shahid] relax _VALID_URL(closes  #28772, closes
 #28930)

---
 youtube_dl/extractor/shahid.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/shahid.py b/youtube_dl/extractor/shahid.py
index b5e093bd2..88b938e05 100644
--- a/youtube_dl/extractor/shahid.py
+++ b/youtube_dl/extractor/shahid.py
@@ -21,6 +21,7 @@ from ..utils import (
 class ShahidBaseIE(AWSIE):
     _AWS_PROXY_HOST = 'api2.shahid.net'
     _AWS_API_KEY = '2RRtuMHx95aNI1Kvtn2rChEuwsCogUd4samGPjLh'
+    _VALID_URL_BASE = r'https?://shahid\.mbc\.net/[a-z]{2}/'
 
     def _handle_error(self, e):
         fail_data = self._parse_json(
@@ -49,7 +50,7 @@ class ShahidBaseIE(AWSIE):
 
 class ShahidIE(ShahidBaseIE):
     _NETRC_MACHINE = 'shahid'
-    _VALID_URL = r'https?://shahid\.mbc\.net/ar/(?:serie|show|movie)s/[^/]+/(?P<type>episode|clip|movie)-(?P<id>\d+)'
+    _VALID_URL = ShahidBaseIE._VALID_URL_BASE + r'(?:serie|show|movie)s/[^/]+/(?P<type>episode|clip|movie)-(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://shahid.mbc.net/ar/shows/%D9%85%D8%AA%D8%AD%D9%81-%D8%A7%D9%84%D8%AF%D8%AD%D9%8A%D8%AD-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D9%83%D9%84%D9%8A%D8%A8-1/clip-816924',
         'info_dict': {
@@ -73,6 +74,9 @@ class ShahidIE(ShahidBaseIE):
         # shahid plus subscriber only
         'url': 'https://shahid.mbc.net/ar/series/%D9%85%D8%B1%D8%A7%D9%8A%D8%A7-2011-%D8%A7%D9%84%D9%85%D9%88%D8%B3%D9%85-1-%D8%A7%D9%84%D8%AD%D9%84%D9%82%D8%A9-1/episode-90511',
         'only_matching': True
+    }, {
+        'url': 'https://shahid.mbc.net/en/shows/Ramez-Fi-Al-Shallal-season-1-episode-1/episode-359319',
+        'only_matching': True
     }]
 
     def _real_initialize(self):
@@ -168,7 +172,7 @@ class ShahidIE(ShahidBaseIE):
 
 
 class ShahidShowIE(ShahidBaseIE):
-    _VALID_URL = r'https?://shahid\.mbc\.net/ar/(?:show|serie)s/[^/]+/(?:show|series)-(?P<id>\d+)'
+    _VALID_URL = ShahidBaseIE._VALID_URL_BASE + r'(?:show|serie)s/[^/]+/(?:show|series)-(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://shahid.mbc.net/ar/shows/%D8%B1%D8%A7%D9%85%D8%B2-%D9%82%D8%B1%D8%B4-%D8%A7%D9%84%D8%A8%D8%AD%D8%B1/show-79187',
         'info_dict': {

From dfbbe2902fc67f0f93ee47a8077c148055c67a9b Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 17 May 2021 12:56:49 +0100
Subject: [PATCH 388/860] [redbulltv] fix embed data extraction(closes #28770)

---
 youtube_dl/extractor/redbulltv.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/redbulltv.py b/youtube_dl/extractor/redbulltv.py
index 3aae79f5d..6d000b372 100644
--- a/youtube_dl/extractor/redbulltv.py
+++ b/youtube_dl/extractor/redbulltv.py
@@ -133,8 +133,10 @@ class RedBullEmbedIE(RedBullTVIE):
         rrn_id = self._match_id(url)
         asset_id = self._download_json(
             'https://edge-graphql.crepo-production.redbullaws.com/v1/graphql',
-            rrn_id, headers={'API-KEY': 'e90a1ff11335423998b100c929ecc866'},
-            query={
+            rrn_id, headers={
+                'Accept': 'application/json',
+                'API-KEY': 'e90a1ff11335423998b100c929ecc866',
+            }, query={
                 'query': '''{
   resource(id: "%s", enforceGeoBlocking: false) {
     %s

From 1980ff4550a3f040fbc1e054d6b91013e9d8cb96 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Wed, 26 May 2021 11:04:39 +0100
Subject: [PATCH 389/860] [vimeo] fix vimeo pro embed extraction(closes #29126)

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 102687b82..0b386f450 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -647,7 +647,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                         expected=True)
             raise
 
-        if '://player.vimeo.com/video/' in url:
+        if '//player.vimeo.com/video/' in url:
             config = self._parse_json(self._search_regex(
                 r'\bconfig\s*=\s*({.+?})\s*;', webpage, 'info section'), video_id)
             if config.get('view') == 4:

From 24297a42efc52862cb9510d32b28efd7faf49af6 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 May 2021 00:36:26 +0700
Subject: [PATCH 390/860] [youtube] Fix get_video_info request (closes #29086,
 closes #29165)

---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0c52e5a8b..bf858c39d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1499,6 +1499,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'unable to download video info webpage', query={
                         'video_id': video_id,
                         'eurl': 'https://youtube.googleapis.com/v/' + video_id,
+                        'html5': 1,
                     }, fatal=False)),
                 lambda x: x['player_response'][0],
                 compat_str) or '{}', video_id)

From e13a01061d149f4fac7db1a50124c4745a11c16e Mon Sep 17 00:00:00 2001
From: phlip <git@mrphlip.com>
Date: Fri, 28 May 2021 11:01:59 +1000
Subject: [PATCH 391/860] [twitch:clips] Add access token query to download
 URLs (closes #29136)

---
 youtube_dl/extractor/twitch.py | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index a7867f4d3..7f9738d43 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -49,6 +49,7 @@ class TwitchBaseIE(InfoExtractor):
         'ChannelCollectionsContent': '07e3691a1bad77a36aba590c351180439a40baefc1c275356f40fc7082419a84',
         'StreamMetadata': '1c719a40e481453e5c48d9bb585d971b8b372f8ebb105b17076722264dfa5b3e',
         'ComscoreStreamingQuery': 'e1edae8122517d013405f237ffcc124515dc6ded82480a88daef69c83b53ac01',
+        'VideoAccessToken_Clip': '36b89d2507fce29e5ca551df756d27c1cfe079e2609642b4390aa4c35796eb11',
         'VideoPreviewOverlay': '3006e77e51b128d838fa4e835723ca4dc9a05c5efd4466c1085215c6e437e65c',
         'VideoMetadata': '226edb3e692509f727fd56821f5653c05740242c82b0388883e0c0e75dcbf687',
     }
@@ -924,6 +925,17 @@ class TwitchClipsIE(TwitchBaseIE):
             raise ExtractorError(
                 'This clip is no longer available', expected=True)
 
+        access_token = self._download_gql(
+            video_id, [{
+                'operationName': 'VideoAccessToken_Clip',
+                'variables': {
+                    'slug': video_id,
+                },
+            }],
+            'Downloading access token GraphQL')
+        access_token = try_get(
+            access_token, lambda x: x[0]['data']['clip']['playbackAccessToken'])
+
         formats = []
         for option in clip.get('videoQualities', []):
             if not isinstance(option, dict):
@@ -931,6 +943,14 @@ class TwitchClipsIE(TwitchBaseIE):
             source = url_or_none(option.get('sourceURL'))
             if not source:
                 continue
+            if access_token:
+                source = "%s%s%s" % (
+                    source,
+                    "&" if "?" in source else "?",
+                    compat_urllib_parse_urlencode({
+                        "sig": access_token.get('signature'),
+                        "token": access_token.get('value'),
+                    }))
             formats.append({
                 'url': source,
                 'format_id': option.get('quality'),

From f3cd1d9cec91943a459a0662cbcffe3b2e1f6675 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 May 2021 01:46:49 +0700
Subject: [PATCH 392/860] [twitch:clips] Improve extraction (closes #29149)

---
 youtube_dl/extractor/twitch.py | 48 ++++++++++++++++------------------
 1 file changed, 23 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/twitch.py b/youtube_dl/extractor/twitch.py
index 7f9738d43..a378bd6dc 100644
--- a/youtube_dl/extractor/twitch.py
+++ b/youtube_dl/extractor/twitch.py
@@ -894,7 +894,25 @@ class TwitchClipsIE(TwitchBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        clip = self._download_base_gql(
+        clip = self._download_gql(
+            video_id, [{
+                'operationName': 'VideoAccessToken_Clip',
+                'variables': {
+                    'slug': video_id,
+                },
+            }],
+            'Downloading clip access token GraphQL')[0]['data']['clip']
+
+        if not clip:
+            raise ExtractorError(
+                'This clip is no longer available', expected=True)
+
+        access_query = {
+            'sig': clip['playbackAccessToken']['signature'],
+            'token': clip['playbackAccessToken']['value'],
+        }
+
+        data = self._download_base_gql(
             video_id, {
                 'query': '''{
   clip(slug: "%s") {
@@ -919,22 +937,10 @@ class TwitchClipsIE(TwitchBaseIE):
     }
     viewCount
   }
-}''' % video_id}, 'Downloading clip GraphQL')['data']['clip']
+}''' % video_id}, 'Downloading clip GraphQL', fatal=False)
 
-        if not clip:
-            raise ExtractorError(
-                'This clip is no longer available', expected=True)
-
-        access_token = self._download_gql(
-            video_id, [{
-                'operationName': 'VideoAccessToken_Clip',
-                'variables': {
-                    'slug': video_id,
-                },
-            }],
-            'Downloading access token GraphQL')
-        access_token = try_get(
-            access_token, lambda x: x[0]['data']['clip']['playbackAccessToken'])
+        if data:
+            clip = try_get(data, lambda x: x['data']['clip'], dict) or clip
 
         formats = []
         for option in clip.get('videoQualities', []):
@@ -943,16 +949,8 @@ class TwitchClipsIE(TwitchBaseIE):
             source = url_or_none(option.get('sourceURL'))
             if not source:
                 continue
-            if access_token:
-                source = "%s%s%s" % (
-                    source,
-                    "&" if "?" in source else "?",
-                    compat_urllib_parse_urlencode({
-                        "sig": access_token.get('signature'),
-                        "token": access_token.get('value'),
-                    }))
             formats.append({
-                'url': source,
+                'url': update_url_query(source, access_query),
                 'format_id': option.get('quality'),
                 'height': int_or_none(option.get('quality')),
                 'fps': int_or_none(option.get('frameRate')),

From 6511b8e8d7db78d4ba3706df5122a74e1c9b9b57 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 May 2021 03:05:22 +0700
Subject: [PATCH 393/860] [ted] Prefer own formats over external sources
 (closes #29142)

---
 youtube_dl/extractor/ted.py | 24 ++++++++++++++----------
 1 file changed, 14 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/ted.py b/youtube_dl/extractor/ted.py
index 63e2455b2..f09f1a3f9 100644
--- a/youtube_dl/extractor/ted.py
+++ b/youtube_dl/extractor/ted.py
@@ -123,6 +123,10 @@ class TEDIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # with own formats and private Youtube external
+        'url': 'https://www.ted.com/talks/spencer_wells_a_family_tree_for_humanity',
+        'only_matching': True,
     }]
 
     _NATIVE_FORMATS = {
@@ -210,16 +214,6 @@ class TEDIE(InfoExtractor):
 
         player_talk = talk_info['player_talks'][0]
 
-        external = player_talk.get('external')
-        if isinstance(external, dict):
-            service = external.get('service')
-            if isinstance(service, compat_str):
-                ext_url = None
-                if service.lower() == 'youtube':
-                    ext_url = external.get('code')
-
-                return self.url_result(ext_url or external['uri'])
-
         resources_ = player_talk.get('resources') or talk_info.get('resources')
 
         http_url = None
@@ -294,6 +288,16 @@ class TEDIE(InfoExtractor):
                 'vcodec': 'none',
             })
 
+        if not formats:
+            external = player_talk.get('external')
+            if isinstance(external, dict):
+                service = external.get('service')
+                if isinstance(service, compat_str):
+                    ext_url = None
+                    if service.lower() == 'youtube':
+                        ext_url = external.get('code')
+                    return self.url_result(ext_url or external['uri'])
+
         self._sort_formats(formats)
 
         video_id = compat_str(talk_info['id'])

From 2ee6c7f11074917c08253af4c47f9258aa1e0dad Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 May 2021 03:43:59 +0700
Subject: [PATCH 394/860] [ustream] Detect https embeds (closes #29133)

---
 youtube_dl/extractor/ustream.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/ustream.py b/youtube_dl/extractor/ustream.py
index 9e860aeb7..1e29cbe22 100644
--- a/youtube_dl/extractor/ustream.py
+++ b/youtube_dl/extractor/ustream.py
@@ -75,7 +75,7 @@ class UstreamIE(InfoExtractor):
     @staticmethod
     def _extract_url(webpage):
         mobj = re.search(
-            r'<iframe[^>]+?src=(["\'])(?P<url>http://(?:www\.)?(?:ustream\.tv|video\.ibm\.com)/embed/.+?)\1', webpage)
+            r'<iframe[^>]+?src=(["\'])(?P<url>https?://(?:www\.)?(?:ustream\.tv|video\.ibm\.com)/embed/.+?)\1', webpage)
         if mobj is not None:
             return mobj.group('url')
 

From d495292852b6c2f1bd58bc2141ff2b0265c952cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 30 May 2021 06:14:59 +0700
Subject: [PATCH 395/860] [ard] Relax _VALID_URL and fix video ids (closes
 #22724, closes #29091)

---
 youtube_dl/extractor/ard.py | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index d57c5ba0f..d45a9fe52 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -249,14 +249,14 @@ class ARDMediathekIE(ARDMediathekBaseIE):
 
 
 class ARDIE(InfoExtractor):
-    _VALID_URL = r'(?P<mainurl>https?://(?:www\.)?daserste\.de/[^?#]+/videos(?:extern)?/(?P<display_id>[^/?#]+)-(?:video-?)?(?P<id>[0-9]+))\.html'
+    _VALID_URL = r'(?P<mainurl>https?://(?:www\.)?daserste\.de/(?:[^/?#&]+/)+(?P<id>[^/?#&]+))\.html'
     _TESTS = [{
         # available till 7.01.2022
         'url': 'https://www.daserste.de/information/talk/maischberger/videos/maischberger-die-woche-video100.html',
         'md5': '867d8aa39eeaf6d76407c5ad1bb0d4c1',
         'info_dict': {
-            'display_id': 'maischberger-die-woche',
-            'id': '100',
+            'id': 'maischberger-die-woche-video100',
+            'display_id': 'maischberger-die-woche-video100',
             'ext': 'mp4',
             'duration': 3687.0,
             'title': 'maischberger. die woche vom 7. Januar 2021',
@@ -264,16 +264,25 @@ class ARDIE(InfoExtractor):
             'thumbnail': r're:^https?://.*\.jpg$',
         },
     }, {
-        'url': 'https://www.daserste.de/information/reportage-dokumentation/erlebnis-erde/videosextern/woelfe-und-herdenschutzhunde-ungleiche-brueder-102.html',
+        'url': 'https://www.daserste.de/information/politik-weltgeschehen/morgenmagazin/videosextern/dominik-kahun-aus-der-nhl-direkt-zur-weltmeisterschaft-100.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.daserste.de/information/nachrichten-wetter/tagesthemen/videosextern/tagesthemen-17736.html',
         'only_matching': True,
     }, {
         'url': 'http://www.daserste.de/information/reportage-dokumentation/dokus/videos/die-story-im-ersten-mission-unter-falscher-flagge-100.html',
         'only_matching': True,
+    }, {
+        'url': 'https://www.daserste.de/unterhaltung/serie/in-aller-freundschaft-die-jungen-aerzte/Drehpause-100.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.daserste.de/unterhaltung/film/filmmittwoch-im-ersten/videos/making-ofwendezeit-video-100.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
+        display_id = mobj.group('id')
 
         player_url = mobj.group('mainurl') + '~playerXml.xml'
         doc = self._download_xml(player_url, display_id)
@@ -324,7 +333,7 @@ class ARDIE(InfoExtractor):
         self._sort_formats(formats)
 
         return {
-            'id': mobj.group('id'),
+            'id': xpath_text(video_node, './videoId', default=display_id),
             'formats': formats,
             'display_id': display_id,
             'title': video_node.find('./title').text,

From 82f3993ba3f4d435d3bc9e37426ab225f5549510 Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Fri, 4 Jun 2021 17:51:44 +0100
Subject: [PATCH 396/860] [formula1] fix extraction(closes #29206)

---
 youtube_dl/extractor/formula1.py | 32 +++++++++++++-------------------
 1 file changed, 13 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/formula1.py b/youtube_dl/extractor/formula1.py
index fecfc28ae..67662e6de 100644
--- a/youtube_dl/extractor/formula1.py
+++ b/youtube_dl/extractor/formula1.py
@@ -5,29 +5,23 @@ from .common import InfoExtractor
 
 
 class Formula1IE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?formula1\.com/(?:content/fom-website/)?en/video/\d{4}/\d{1,2}/(?P<id>.+?)\.html'
-    _TESTS = [{
-        'url': 'http://www.formula1.com/content/fom-website/en/video/2016/5/Race_highlights_-_Spain_2016.html',
-        'md5': '8c79e54be72078b26b89e0e111c0502b',
+    _VALID_URL = r'https?://(?:www\.)?formula1\.com/en/latest/video\.[^.]+\.(?P<id>\d+)\.html'
+    _TEST = {
+        'url': 'https://www.formula1.com/en/latest/video.race-highlights-spain-2016.6060988138001.html',
+        'md5': 'be7d3a8c2f804eb2ab2aa5d941c359f8',
         'info_dict': {
-            'id': 'JvYXJpMzE6pArfHWm5ARp5AiUmD-gibV',
+            'id': '6060988138001',
             'ext': 'mp4',
             'title': 'Race highlights - Spain 2016',
+            'timestamp': 1463332814,
+            'upload_date': '20160515',
+            'uploader_id': '6057949432001',
         },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-        'add_ie': ['Ooyala'],
-    }, {
-        'url': 'http://www.formula1.com/en/video/2016/5/Race_highlights_-_Spain_2016.html',
-        'only_matching': True,
-    }]
+        'add_ie': ['BrightcoveNew'],
+    }
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/6057949432001/S1WMrhjlh_default/index.html?videoId=%s'
 
     def _real_extract(self, url):
-        display_id = self._match_id(url)
-        webpage = self._download_webpage(url, display_id)
-        ooyala_embed_code = self._search_regex(
-            r'data-videoid="([^"]+)"', webpage, 'ooyala embed code')
+        bc_id = self._match_id(url)
         return self.url_result(
-            'ooyala:%s' % ooyala_embed_code, 'Ooyala', ooyala_embed_code)
+            self.BRIGHTCOVE_URL_TEMPLATE % bc_id, 'BrightcoveNew', bc_id)

From 943070af4a9e13ef2b81c5e484d9c799f1845aab Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sat, 5 Jun 2021 23:42:25 +0700
Subject: [PATCH 397/860] [orf:tvthek] Fix thumbnails extraction (closes
 #29217)

---
 youtube_dl/extractor/orf.py | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 3fadbcbea..ed8a9a841 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -140,6 +140,25 @@ class ORFTVthekIE(InfoExtractor):
                 })
 
             upload_date = unified_strdate(sd.get('created_date'))
+
+            thumbnails = []
+            preview = sd.get('preview_image_url')
+            if preview:
+                thumbnails.append({
+                    'id': 'preview',
+                    'url': preview,
+                    'preference': 0,
+                })
+            image = sd.get('image_full_url')
+            if not image and len(data_jsb) == 1:
+                image = self._og_search_thumbnail(webpage)
+            if image:
+                thumbnails.append({
+                    'id': 'full',
+                    'url': image,
+                    'preference': 1,
+                })
+
             entries.append({
                 '_type': 'video',
                 'id': video_id,
@@ -149,7 +168,7 @@ class ORFTVthekIE(InfoExtractor):
                 'description': sd.get('description'),
                 'duration': int_or_none(sd.get('duration_in_seconds')),
                 'upload_date': upload_date,
-                'thumbnail': sd.get('image_full_url'),
+                'thumbnails': thumbnails,
             })
 
         return {

From fdf91c52a8b58b3b7c12a393629fc962d6ab7618 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Jun 2021 00:11:09 +0700
Subject: [PATCH 398/860] [youporn] Fix formats and view count extraction
 (closes #29216)

---
 youtube_dl/extractor/youporn.py | 111 ++++++++++++--------------------
 1 file changed, 42 insertions(+), 69 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 33114363d..7084d3d12 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -4,13 +4,12 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    extract_attributes,
     int_or_none,
     str_to_int,
-    unescapeHTML,
     unified_strdate,
     url_or_none,
 )
-from ..aes import aes_decrypt_text
 
 
 class YouPornIE(InfoExtractor):
@@ -34,6 +33,7 @@ class YouPornIE(InfoExtractor):
             'tags': list,
             'age_limit': 18,
         },
+        'skip': 'This video has been disabled',
     }, {
         # Unknown uploader
         'url': 'http://www.youporn.com/watch/561726/big-tits-awesome-brunette-on-amazing-webcam-show/?from=related3&al=2&from_id=561726&pos=4',
@@ -78,6 +78,40 @@ class YouPornIE(InfoExtractor):
         video_id = mobj.group('id')
         display_id = mobj.group('display_id') or video_id
 
+        definitions = self._download_json(
+            'https://www.youporn.com/api/video/media_definitions/%s/' % video_id,
+            display_id)
+
+        formats = []
+        for definition in definitions:
+            if not isinstance(definition, dict):
+                continue
+            video_url = url_or_none(definition.get('videoUrl'))
+            if not video_url:
+                continue
+            f = {
+                'url': video_url,
+                'filesize': int_or_none(definition.get('videoSize')),
+            }
+            height = int_or_none(definition.get('quality'))
+            # Video URL's path looks like this:
+            #  /201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
+            #  /201012/17/505835/vl_240p_240k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
+            #  /videos/201703/11/109285532/1080P_4000K_109285532.mp4
+            # We will benefit from it by extracting some metadata
+            mobj = re.search(r'(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+', video_url)
+            if mobj:
+                if not height:
+                    height = int(mobj.group('height'))
+                bitrate = int(mobj.group('bitrate'))
+                f.update({
+                    'format_id': '%dp-%dk' % (height, bitrate),
+                    'tbr': bitrate,
+                })
+            f['height'] = height
+            formats.append(f)
+        self._sort_formats(formats)
+
         webpage = self._download_webpage(
             'http://www.youporn.com/watch/%s' % video_id, display_id,
             headers={'Cookie': 'age_verified=1'})
@@ -88,65 +122,6 @@ class YouPornIE(InfoExtractor):
             webpage, default=None) or self._html_search_meta(
             'title', webpage, fatal=True)
 
-        links = []
-
-        # Main source
-        definitions = self._parse_json(
-            self._search_regex(
-                r'mediaDefinition\s*[=:]\s*(\[.+?\])\s*[;,]', webpage,
-                'media definitions', default='[]'),
-            video_id, fatal=False)
-        if definitions:
-            for definition in definitions:
-                if not isinstance(definition, dict):
-                    continue
-                video_url = url_or_none(definition.get('videoUrl'))
-                if video_url:
-                    links.append(video_url)
-
-        # Fallback #1, this also contains extra low quality 180p format
-        for _, link in re.findall(r'<a[^>]+href=(["\'])(http(?:(?!\1).)+\.mp4(?:(?!\1).)*)\1[^>]+title=["\']Download [Vv]ideo', webpage):
-            links.append(link)
-
-        # Fallback #2 (unavailable as at 22.06.2017)
-        sources = self._search_regex(
-            r'(?s)sources\s*:\s*({.+?})', webpage, 'sources', default=None)
-        if sources:
-            for _, link in re.findall(r'[^:]+\s*:\s*(["\'])(http.+?)\1', sources):
-                links.append(link)
-
-        # Fallback #3 (unavailable as at 22.06.2017)
-        for _, link in re.findall(
-                r'(?:videoSrc|videoIpadUrl|html5PlayerSrc)\s*[:=]\s*(["\'])(http.+?)\1', webpage):
-            links.append(link)
-
-        # Fallback #4, encrypted links (unavailable as at 22.06.2017)
-        for _, encrypted_link in re.findall(
-                r'encryptedQuality\d{3,4}URL\s*=\s*(["\'])([\da-zA-Z+/=]+)\1', webpage):
-            links.append(aes_decrypt_text(encrypted_link, title, 32).decode('utf-8'))
-
-        formats = []
-        for video_url in set(unescapeHTML(link) for link in links):
-            f = {
-                'url': video_url,
-            }
-            # Video URL's path looks like this:
-            #  /201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
-            #  /201012/17/505835/vl_240p_240k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
-            #  /videos/201703/11/109285532/1080P_4000K_109285532.mp4
-            # We will benefit from it by extracting some metadata
-            mobj = re.search(r'(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+', video_url)
-            if mobj:
-                height = int(mobj.group('height'))
-                bitrate = int(mobj.group('bitrate'))
-                f.update({
-                    'format_id': '%dp-%dk' % (height, bitrate),
-                    'height': height,
-                    'tbr': bitrate,
-                })
-            formats.append(f)
-        self._sort_formats(formats)
-
         description = self._html_search_regex(
             r'(?s)<div[^>]+\bid=["\']description["\'][^>]*>(.+?)</div>',
             webpage, 'description',
@@ -169,13 +144,12 @@ class YouPornIE(InfoExtractor):
 
         age_limit = self._rta_search(webpage)
 
-        average_rating = int_or_none(self._search_regex(
-            r'<div[^>]+class=["\']videoRatingPercentage["\'][^>]*>(\d+)%</div>',
-            webpage, 'average rating', fatal=False))
-
-        view_count = str_to_int(self._search_regex(
-            r'(?s)<div[^>]+class=(["\']).*?\bvideoInfoViews\b.*?\1[^>]*>.*?(?P<count>[\d,.]+)<',
-            webpage, 'view count', fatal=False, group='count'))
+        view_count = None
+        views = self._search_regex(
+            r'(<div[^>]+\bclass=["\']js_videoInfoViews["\']>)', webpage,
+            'views', default=None)
+        if views:
+            view_count = str_to_int(extract_attributes(views).get('data-value'))
         comment_count = str_to_int(self._search_regex(
             r'>All [Cc]omments? \(([\d,.]+)\)',
             webpage, 'comment count', default=None))
@@ -201,7 +175,6 @@ class YouPornIE(InfoExtractor):
             'duration': duration,
             'uploader': uploader,
             'upload_date': upload_date,
-            'average_rating': average_rating,
             'view_count': view_count,
             'comment_count': comment_count,
             'categories': categories,

From bb7ac1ed669d67d79fa1a3b9e5c70271892ecbcc Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Jun 2021 01:16:43 +0700
Subject: [PATCH 399/860] [facebook] Improve login required detection

---
 youtube_dl/extractor/facebook.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/facebook.py b/youtube_dl/extractor/facebook.py
index cb34c59f5..04650af39 100644
--- a/youtube_dl/extractor/facebook.py
+++ b/youtube_dl/extractor/facebook.py
@@ -521,7 +521,10 @@ class FacebookIE(InfoExtractor):
                 raise ExtractorError(
                     'The video is not available, Facebook said: "%s"' % m_msg.group(1),
                     expected=True)
-            elif '>You must log in to continue' in webpage:
+            elif any(p in webpage for p in (
+                    '>You must log in to continue',
+                    'id="login_form"',
+                    'id="loginbutton"')):
                 self.raise_login_required()
 
         if not video_data and '/watchparty/' in url:

From 5f85eb820cb7eb89dcb567f9cbfefb5d9038b9c0 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Jun 2021 01:32:15 +0700
Subject: [PATCH 400/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 18 ++++++++++++++++++
 1 file changed, 18 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 5ea1d3150..06efe32ab 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,21 @@
+version <unreleased>
+
+Extractors
+* [facebook] Improve login required detection
+* [youporn] Fix formats and view count extraction (#29216)
+* [orf:tvthek] Fix thumbnails extraction (#29217)
+* [formula1] Fix extraction (#29206)
+* [ard] Relax URL regular expression and fix video ids (#22724, #29091)
++ [ustream] Detect https embeds (#29133)
+* [ted] Prefer own formats over external sources (#29142)
+* [twitch:clips] Improve extraction (#29149)
++ [twitch:clips] Add access token query to download URLs (#29136)
+* [youtube] Fix get_video_info request (#29086, #29165)
+* [vimeo] Fix vimeo pro embed extraction (#29126)
+* [redbulltv] Fix embed data extraction (#28770)
+* [shahid] Relax URL regular expression (#28772, #28930)
+
+
 version 2021.05.16
 
 Core

From b224cf39d53bd16bcfda2ac493712c3ff449ecb8 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Sun, 6 Jun 2021 01:38:22 +0700
Subject: [PATCH 401/860] release 2021.06.06

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 youtube_dl/version.py                            | 2 +-
 7 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index d67bb482c..4eb505231 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
+- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.05.16
+ [debug] youtube-dl version 2021.06.06
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index efe9fef8c..9fed0b489 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
+- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index e213fc1a9..573e8ded0 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
+- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index 1645087ad..c0031bf7a 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
+- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.05.16
+ [debug] youtube-dl version 2021.06.06
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index e6e569af6..1138ab2ca 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.05.16. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.05.16**
+- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index 06efe32ab..680fffdf8 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.06.06
 
 Extractors
 * [facebook] Improve login required detection
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b16a84100..461dd87ca 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.05.16'
+__version__ = '2021.06.06'

From c2350cac243ba1ec1586fe85b0d62d1b700047a2 Mon Sep 17 00:00:00 2001
From: Sergey M <dstftw@gmail.com>
Date: Sun, 6 Jun 2021 05:32:27 +0700
Subject: [PATCH 402/860] [README.md] Update MSVC 2010 redist URL (closes
 #29222)

---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 059141611..2841ed68f 100644
--- a/README.md
+++ b/README.md
@@ -893,7 +893,7 @@ Since June 2012 ([#342](https://github.com/ytdl-org/youtube-dl/issues/342)) yout
 
 ### The exe throws an error due to missing `MSVCR100.dll`
 
-To run the exe you need to install first the [Microsoft Visual C++ 2010 Redistributable Package (x86)](https://www.microsoft.com/en-US/download/details.aspx?id=5555).
+To run the exe you need to install first the [Microsoft Visual C++ 2010 Service Pack 1 Redistributable Package (x86)](https://download.microsoft.com/download/1/6/5/165255E7-1014-4D0A-B094-B6A430A6BFFC/vcredist_x86.exe).
 
 ### On Windows, how should I set up ffmpeg and youtube-dl? Where should I put the exe files?
 

From d156bc8d59dd469bf70b822926504f213ce237de Mon Sep 17 00:00:00 2001
From: kikuyan <kikuyan@users.noreply.github.com>
Date: Thu, 17 Jun 2021 06:02:06 +0900
Subject: [PATCH 403/860] [orf:tvthek] Add support for MPD formats (closes
 #28672) (#29236)

---
 youtube_dl/extractor/orf.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index ed8a9a841..8d537d7ae 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -98,6 +98,9 @@ class ORFTVthekIE(InfoExtractor):
                 elif ext == 'f4m':
                     formats.extend(self._extract_f4m_formats(
                         src, video_id, f4m_id=format_id, fatal=False))
+                elif ext == 'mpd':
+                    formats.extend(self._extract_mpd_formats(
+                        src, video_id, mpd_id=format_id, fatal=False))
                 else:
                     formats.append({
                         'format_id': format_id,

From 8fe5d54eb721f1bbb8c8a0d18810a42d1257e406 Mon Sep 17 00:00:00 2001
From: kikuyan <kikuyan@users.noreply.github.com>
Date: Thu, 17 Jun 2021 06:12:13 +0900
Subject: [PATCH 404/860] [appleconnect] Fix extraction (#29208)

---
 youtube_dl/extractor/appleconnect.py | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/appleconnect.py b/youtube_dl/extractor/appleconnect.py
index a84b8b1eb..494f8330c 100644
--- a/youtube_dl/extractor/appleconnect.py
+++ b/youtube_dl/extractor/appleconnect.py
@@ -9,10 +9,10 @@ from ..utils import (
 
 
 class AppleConnectIE(InfoExtractor):
-    _VALID_URL = r'https?://itunes\.apple\.com/\w{0,2}/?post/idsa\.(?P<id>[\w-]+)'
-    _TEST = {
+    _VALID_URL = r'https?://itunes\.apple\.com/\w{0,2}/?post/(?:id)?sa\.(?P<id>[\w-]+)'
+    _TESTS = [{
         'url': 'https://itunes.apple.com/us/post/idsa.4ab17a39-2720-11e5-96c5-a5b38f6c42d3',
-        'md5': 'e7c38568a01ea45402570e6029206723',
+        'md5': 'c1d41f72c8bcaf222e089434619316e4',
         'info_dict': {
             'id': '4ab17a39-2720-11e5-96c5-a5b38f6c42d3',
             'ext': 'm4v',
@@ -22,7 +22,10 @@ class AppleConnectIE(InfoExtractor):
             'upload_date': '20150710',
             'timestamp': 1436545535,
         },
-    }
+    }, {
+        'url': 'https://itunes.apple.com/us/post/sa.0fe0229f-2457-11e5-9f40-1bb645f2d5d9',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -36,7 +39,7 @@ class AppleConnectIE(InfoExtractor):
 
         video_data = self._parse_json(video_json, video_id)
         timestamp = str_to_int(self._html_search_regex(r'data-timestamp="(\d+)"', webpage, 'timestamp'))
-        like_count = str_to_int(self._html_search_regex(r'(\d+) Loves', webpage, 'like count'))
+        like_count = str_to_int(self._html_search_regex(r'(\d+) Loves', webpage, 'like count', default=None))
 
         return {
             'id': video_id,

From a7f61feab2dbfc50a7ebe8b0ea390bd0e5edf77a Mon Sep 17 00:00:00 2001
From: kikuyan <kikuyan@users.noreply.github.com>
Date: Thu, 17 Jun 2021 12:34:33 +0900
Subject: [PATCH 405/860] [egghead] Add support for app.egghead.io (closes
 #28404) (#29303)

Co-authored-by: Sergey M. <dstftw@gmail.com>
---
 youtube_dl/extractor/egghead.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/egghead.py b/youtube_dl/extractor/egghead.py
index aff9b88c0..9bbd703e0 100644
--- a/youtube_dl/extractor/egghead.py
+++ b/youtube_dl/extractor/egghead.py
@@ -22,16 +22,19 @@ class EggheadBaseIE(InfoExtractor):
 class EggheadCourseIE(EggheadBaseIE):
     IE_DESC = 'egghead.io course'
     IE_NAME = 'egghead:course'
-    _VALID_URL = r'https://egghead\.io/courses/(?P<id>[^/?#&]+)'
-    _TEST = {
+    _VALID_URL = r'https://(?:app\.)?egghead\.io/(?:course|playlist)s/(?P<id>[^/?#&]+)'
+    _TESTS = [{
         'url': 'https://egghead.io/courses/professor-frisby-introduces-composable-functional-javascript',
         'playlist_count': 29,
         'info_dict': {
-            'id': '72',
+            'id': '432655',
             'title': 'Professor Frisby Introduces Composable Functional JavaScript',
             'description': 're:(?s)^This course teaches the ubiquitous.*You\'ll start composing functionality before you know it.$',
         },
-    }
+    }, {
+        'url': 'https://app.egghead.io/playlists/professor-frisby-introduces-composable-functional-javascript',
+        'only_matching': True,
+    }]
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
@@ -65,7 +68,7 @@ class EggheadCourseIE(EggheadBaseIE):
 class EggheadLessonIE(EggheadBaseIE):
     IE_DESC = 'egghead.io lesson'
     IE_NAME = 'egghead:lesson'
-    _VALID_URL = r'https://egghead\.io/(?:api/v1/)?lessons/(?P<id>[^/?#&]+)'
+    _VALID_URL = r'https://(?:app\.)?egghead\.io/(?:api/v1/)?lessons/(?P<id>[^/?#&]+)'
     _TESTS = [{
         'url': 'https://egghead.io/lessons/javascript-linear-data-flow-with-container-style-types-box',
         'info_dict': {
@@ -88,6 +91,9 @@ class EggheadLessonIE(EggheadBaseIE):
     }, {
         'url': 'https://egghead.io/api/v1/lessons/react-add-redux-to-a-react-application',
         'only_matching': True,
+    }, {
+        'url': 'https://app.egghead.io/lessons/javascript-linear-data-flow-with-container-style-types-box',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From 3a7ef27cf306a0a8f79ebd78ae60329c53080d14 Mon Sep 17 00:00:00 2001
From: kikuyan <kikuyan@users.noreply.github.com>
Date: Mon, 21 Jun 2021 01:58:19 +0900
Subject: [PATCH 406/860] [postprocessor/ffmpeg] Show ffmpeg output on error
 (refs #22680) (#29336)

---
 youtube_dl/postprocessor/ffmpeg.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 5f7298345..9f76c9d4e 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -231,7 +231,10 @@ class FFmpegPostProcessor(PostProcessor):
         stdout, stderr = p.communicate()
         if p.returncode != 0:
             stderr = stderr.decode('utf-8', 'replace')
-            msg = stderr.strip().split('\n')[-1]
+            msgs = stderr.strip().split('\n')
+            msg = msgs[-1]
+            if self._downloader.params.get('verbose', False):
+                self._downloader.to_screen('[debug] ' + '\n'.join(msgs[:-1]))
             raise FFmpegPostProcessorError(msg)
         self.try_utime(out_path, oldest_mtime, oldest_mtime)
 

From 57b9a4b4c6cf2580b5007db78bd333a9a237fd47 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Jun 2021 00:36:28 +0700
Subject: [PATCH 407/860] [nrk] Switch psapi URL to https (closes #29344)

Catalog calls no longer work via http
---
 youtube_dl/extractor/nrk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 40dee2162..6d01a25c3 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -58,7 +58,7 @@ class NRKBaseIE(InfoExtractor):
 
     def _call_api(self, path, video_id, item=None, note=None, fatal=True, query=None):
         return self._download_json(
-            urljoin('http://psapi.nrk.no/', path),
+            urljoin('https://psapi.nrk.no/', path),
             video_id, note or 'Downloading %s JSON' % item,
             fatal=fatal, query=query,
             headers={'Accept-Encoding': 'gzip, deflate, br'})

From cc21aebe9071660ba558dae75c3066a4a3b38820 Mon Sep 17 00:00:00 2001
From: Logan B <einstein95@users.noreply.github.com>
Date: Mon, 21 Jun 2021 05:41:14 +1200
Subject: [PATCH 408/860] [umg:de] Update GraphQL API URL (#29304)

Previous one no longer resolves

Co-authored-by: Sergey M. <dstftw@gmail.com>
---
 youtube_dl/extractor/umg.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/umg.py b/youtube_dl/extractor/umg.py
index d815cd9a6..47948b6ce 100644
--- a/youtube_dl/extractor/umg.py
+++ b/youtube_dl/extractor/umg.py
@@ -28,7 +28,7 @@ class UMGDeIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
         video_data = self._download_json(
-            'https://api.universal-music.de/graphql',
+            'https://graphql.universal-music.de/',
             video_id, query={
                 'query': '''{
   universalMusic(channel:16) {
@@ -56,11 +56,9 @@ class UMGDeIE(InfoExtractor):
         formats = []
 
         def add_m3u8_format(format_id):
-            m3u8_formats = self._extract_m3u8_formats(
+            formats.extend(self._extract_m3u8_formats(
                 hls_url_template % format_id, video_id, 'mp4',
-                'm3u8_native', m3u8_id='hls', fatal='False')
-            if m3u8_formats and m3u8_formats[0].get('height'):
-                formats.extend(m3u8_formats)
+                'm3u8_native', m3u8_id='hls', fatal=False))
 
         for f in video_data.get('formats', []):
             f_url = f.get('url')

From 41317030017418c89742594a80c0596c4b26bbb9 Mon Sep 17 00:00:00 2001
From: bopol <bopol@e.email>
Date: Sun, 20 Jun 2021 19:42:09 +0200
Subject: [PATCH 409/860] [youtube] Update invidious instance list (#29281)

---
 youtube_dl/extractor/youtube.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index bf858c39d..35058950a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -353,7 +353,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         r'(?:www\.)?invidious\.13ad\.de',
         r'(?:www\.)?invidious\.mastodon\.host',
         r'(?:www\.)?invidious\.zapashcanon\.fr',
-        r'(?:www\.)?invidious\.kavin\.rocks',
+        r'(?:www\.)?(?:invidious(?:-us)?|piped)\.kavin\.rocks',
         r'(?:www\.)?invidious\.tinfoil-hat\.net',
         r'(?:www\.)?invidious\.himiko\.cloud',
         r'(?:www\.)?invidious\.reallyancient\.tech',
@@ -380,6 +380,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         r'(?:www\.)?invidious\.toot\.koeln',
         r'(?:www\.)?invidious\.fdn\.fr',
         r'(?:www\.)?watch\.nettohikari\.com',
+        r'(?:www\.)?invidious\.namazso\.eu',
+        r'(?:www\.)?invidious\.silkky\.cloud',
+        r'(?:www\.)?invidious\.exonip\.de',
+        r'(?:www\.)?invidious\.riverside\.rocks',
+        r'(?:www\.)?invidious\.blamefran\.net',
+        r'(?:www\.)?invidious\.moomoo\.de',
+        r'(?:www\.)?ytb\.trom\.tf',
+        r'(?:www\.)?yt\.cyberhost\.uk',
         r'(?:www\.)?kgg2m7yk5aybusll\.onion',
         r'(?:www\.)?qklhadlycap4cnod\.onion',
         r'(?:www\.)?axqzx4s6s54s32yentfqojs3x5i7faxza6xo3ehd4bzzsg2ii4fv2iid\.onion',
@@ -388,6 +396,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         r'(?:www\.)?invidious\.l4qlywnpwqsluw65ts7md3khrivpirse744un3x7mlskqauz5pyuzgqd\.onion',
         r'(?:www\.)?owxfohz4kjyv25fvlqilyxast7inivgiktls3th44jhk3ej3i7ya\.b32\.i2p',
         r'(?:www\.)?4l2dgddgsrkf2ous66i6seeyi6etzfgrue332grh2n7madpwopotugyd\.onion',
+        r'(?:www\.)?w6ijuptxiku4xpnnaetxvnkc5vqcdu7mgns2u77qefoixi63vbvnpnqd\.onion',
+        r'(?:www\.)?kbjggqkzv65ivcqj6bumvp337z6264huv5kpkwuv6gu5yjiskvan7fad\.onion',
+        r'(?:www\.)?grwp24hodrefzvjjuccrkw3mjq4tzhaaq32amf33dzpmuxe7ilepcmad\.onion',
+        r'(?:www\.)?hpniueoejy4opn7bc4ftgazyqjoeqwlvh2uiku2xqku6zpoa4bf5ruid\.onion',
     )
     _VALID_URL = r"""(?x)^
                      (

From 4c77a2e538fb23da116aaba0f51e314ef76feb68 Mon Sep 17 00:00:00 2001
From: Tianyi Shi <ShiTianyi2001@outlook.com>
Date: Sun, 20 Jun 2021 19:03:21 +0100
Subject: [PATCH 410/860] [bilibili] Strip uploader name (#29202)

---
 youtube_dl/extractor/bilibili.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index 589fdc1ce..bff6ea194 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -233,7 +233,7 @@ class BiliBiliIE(InfoExtractor):
             webpage)
         if uploader_mobj:
             info.update({
-                'uploader': uploader_mobj.group('name'),
+                'uploader': uploader_mobj.group('name').strip(),
                 'uploader_id': uploader_mobj.group('id'),
             })
         if not info.get('uploader'):

From 03ab02730f77da5b7ad05ca78ff1624d8226ec5f Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Jun 2021 01:34:27 +0700
Subject: [PATCH 411/860] [youtube] Workaround for get_video_info request (refs
 #29333)

See https://github.com/ytdl-org/youtube-dl/issues/29333#issuecomment-864049544
---
 youtube_dl/extractor/youtube.py | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 35058950a..e68214008 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1512,6 +1512,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         'video_id': video_id,
                         'eurl': 'https://youtube.googleapis.com/v/' + video_id,
                         'html5': 1,
+                        # See https://github.com/ytdl-org/youtube-dl/issues/29333#issuecomment-864049544
+                        'c': 'TVHTML5',
+                        'cver': '6.20180913',
                     }, fatal=False)),
                 lambda x: x['player_response'][0],
                 compat_str) or '{}', video_id)

From 47f2f2fbe9730b041b91451d17279216f311ffc2 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Jun 2021 01:35:21 +0700
Subject: [PATCH 412/860] [youtube] Make get_video_info processing more robust
 (closes #29333)

---
 youtube_dl/extractor/youtube.py | 35 ++++++++++++++++++---------------
 1 file changed, 19 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e68214008..dc4bd4a77 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1504,22 +1504,25 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         playability_status = player_response.get('playabilityStatus') or {}
         if playability_status.get('reason') == 'Sign in to confirm your age':
-            pr = self._parse_json(try_get(compat_parse_qs(
-                self._download_webpage(
-                    base_url + 'get_video_info', video_id,
-                    'Refetching age-gated info webpage',
-                    'unable to download video info webpage', query={
-                        'video_id': video_id,
-                        'eurl': 'https://youtube.googleapis.com/v/' + video_id,
-                        'html5': 1,
-                        # See https://github.com/ytdl-org/youtube-dl/issues/29333#issuecomment-864049544
-                        'c': 'TVHTML5',
-                        'cver': '6.20180913',
-                    }, fatal=False)),
-                lambda x: x['player_response'][0],
-                compat_str) or '{}', video_id)
-            if pr:
-                player_response = pr
+            video_info = self._download_webpage(
+                base_url + 'get_video_info', video_id,
+                'Refetching age-gated info webpage',
+                'unable to download video info webpage', query={
+                    'video_id': video_id,
+                    'eurl': 'https://youtube.googleapis.com/v/' + video_id,
+                    'html5': 1,
+                    # See https://github.com/ytdl-org/youtube-dl/issues/29333#issuecomment-864049544
+                    'c': 'TVHTML5',
+                    'cver': '6.20180913',
+                }, fatal=False)
+            if video_info:
+                pr = self._parse_json(
+                    try_get(
+                        compat_parse_qs(video_info),
+                        lambda x: x['player_response'][0], compat_str) or '{}',
+                    video_id, fatal=False)
+                if pr and isinstance(pr, dict):
+                    player_response = pr
 
         trailer_video_id = try_get(
             playability_status,

From 2ccee8db74c36eb1254cdffd4e691e56c0ce0724 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Jun 2021 01:54:52 +0700
Subject: [PATCH 413/860] [curiositystream:collection] Extend _VALID_URL
 (closes #26326, closes #29117)

---
 youtube_dl/extractor/curiositystream.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/curiositystream.py b/youtube_dl/extractor/curiositystream.py
index ae64a07d7..48ff30432 100644
--- a/youtube_dl/extractor/curiositystream.py
+++ b/youtube_dl/extractor/curiositystream.py
@@ -145,7 +145,7 @@ class CuriosityStreamIE(CuriosityStreamBaseIE):
 
 class CuriosityStreamCollectionIE(CuriosityStreamBaseIE):
     IE_NAME = 'curiositystream:collection'
-    _VALID_URL = r'https?://(?:app\.)?curiositystream\.com/(?:collection|series)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:app\.)?curiositystream\.com/(?:collections?|series)/(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://app.curiositystream.com/collection/2',
         'info_dict': {
@@ -157,6 +157,9 @@ class CuriosityStreamCollectionIE(CuriosityStreamBaseIE):
     }, {
         'url': 'https://curiositystream.com/series/2',
         'only_matching': True,
+    }, {
+        'url': 'https://curiositystream.com/collections/36',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):

From da32828208743c8012c8eea01780cbf9b3f60436 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Jun 2021 03:22:37 +0700
Subject: [PATCH 414/860] [pornhub] Dismiss tbr extracted from download URLs
 (closes #28927)

No longer reliable
---
 youtube_dl/extractor/pornhub.py | 14 ++++++--------
 1 file changed, 6 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 031454600..10516ee5a 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -408,17 +408,14 @@ class PornHubIE(PornHubBaseIE):
                     format_url, video_id, 'mp4', entry_protocol='m3u8_native',
                     m3u8_id='hls', fatal=False))
                 return
-            tbr = None
-            mobj = re.search(r'(?P<height>\d+)[pP]?_(?P<tbr>\d+)[kK]', format_url)
-            if mobj:
-                if not height:
-                    height = int(mobj.group('height'))
-                tbr = int(mobj.group('tbr'))
+            if not height:
+                height = int_or_none(self._search_regex(
+                    r'(?P<height>\d+)[pP]?_\d+[kK]', format_url, 'height',
+                    default=None))
             formats.append({
                 'url': format_url,
                 'format_id': '%dp' % height if height else None,
                 'height': height,
-                'tbr': tbr,
             })
 
         for video_url, height in video_urls:
@@ -440,7 +437,8 @@ class PornHubIE(PornHubBaseIE):
                         add_format(video_url, height)
                 continue
             add_format(video_url)
-        self._sort_formats(formats)
+        self._sort_formats(
+            formats, field_preference=('height', 'width', 'fps', 'format_id'))
 
         video_uploader = self._html_search_regex(
             r'(?s)From:&nbsp;.+?<(?:a\b[^>]+\bhref=["\']/(?:(?:user|channel)s|model|pornstar)/|span\b[^>]+\bclass=["\']username)[^>]+>(.+?)<',

From 751c9ae39a0bb9c66eca888a12595624db00bf16 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Jun 2021 03:33:43 +0700
Subject: [PATCH 415/860] [pornhub] Detect geo restriction

---
 youtube_dl/extractor/pornhub.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index 10516ee5a..d74e69ed9 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -236,6 +236,10 @@ class PornHubIE(PornHubBaseIE):
     }, {
         'url': 'https://www.pornhubpremium.com/view_video.php?viewkey=ph5f75b0f4b18e3',
         'only_matching': True,
+    }, {
+        # geo restricted
+        'url': 'https://www.pornhub.com/view_video.php?viewkey=ph5a9813bfa7156',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -275,6 +279,11 @@ class PornHubIE(PornHubBaseIE):
                 'PornHub said: %s' % error_msg,
                 expected=True, video_id=video_id)
 
+        if any(re.search(p, webpage) for p in (
+                r'class=["\']geoBlocked["\']',
+                r'>\s*This content is unavailable in your country')):
+            self.raise_geo_restricted()
+
         # video_title from flashvars contains whitespace instead of non-ASCII (see
         # http://www.pornhub.com/view_video.php?viewkey=1331683002), not relying
         # on that anymore.

From cb668eb973b8f09152bb48e3b49a014d3cb72b22 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Mon, 21 Jun 2021 04:08:15 +0700
Subject: [PATCH 416/860] [pornhub] Add support for pornhubthbh7ap3u.onion

---
 youtube_dl/extractor/pornhub.py | 25 ++++++++++++++++++++-----
 1 file changed, 20 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/pornhub.py b/youtube_dl/extractor/pornhub.py
index d74e69ed9..e2e1500ff 100644
--- a/youtube_dl/extractor/pornhub.py
+++ b/youtube_dl/extractor/pornhub.py
@@ -30,6 +30,7 @@ from ..utils import (
 
 class PornHubBaseIE(InfoExtractor):
     _NETRC_MACHINE = 'pornhub'
+    _PORNHUB_HOST_RE = r'(?:(?P<host>pornhub(?:premium)?\.(?:com|net|org))|pornhubthbh7ap3u\.onion)'
 
     def _download_webpage_handle(self, *args, **kwargs):
         def dl(*args, **kwargs):
@@ -122,11 +123,13 @@ class PornHubIE(PornHubBaseIE):
     _VALID_URL = r'''(?x)
                     https?://
                         (?:
-                            (?:[^/]+\.)?(?P<host>pornhub(?:premium)?\.(?:com|net|org))/(?:(?:view_video\.php|video/show)\?viewkey=|embed/)|
+                            (?:[^/]+\.)?
+                            %s
+                            /(?:(?:view_video\.php|video/show)\?viewkey=|embed/)|
                             (?:www\.)?thumbzilla\.com/video/
                         )
                         (?P<id>[\da-z]+)
-                    '''
+                    ''' % PornHubBaseIE._PORNHUB_HOST_RE
     _TESTS = [{
         'url': 'http://www.pornhub.com/view_video.php?viewkey=648719015',
         'md5': 'a6391306d050e4547f62b3f485dd9ba9',
@@ -240,6 +243,9 @@ class PornHubIE(PornHubBaseIE):
         # geo restricted
         'url': 'https://www.pornhub.com/view_video.php?viewkey=ph5a9813bfa7156',
         'only_matching': True,
+    }, {
+        'url': 'http://pornhubthbh7ap3u.onion/view_video.php?viewkey=ph5a9813bfa7156',
+        'only_matching': True,
     }]
 
     @staticmethod
@@ -520,7 +526,7 @@ class PornHubPlaylistBaseIE(PornHubBaseIE):
 
 
 class PornHubUserIE(PornHubPlaylistBaseIE):
-    _VALID_URL = r'(?P<url>https?://(?:[^/]+\.)?(?P<host>pornhub(?:premium)?\.(?:com|net|org))/(?:(?:user|channel)s|model|pornstar)/(?P<id>[^/?#&]+))(?:[?#&]|/(?!videos)|$)'
+    _VALID_URL = r'(?P<url>https?://(?:[^/]+\.)?%s/(?:(?:user|channel)s|model|pornstar)/(?P<id>[^/?#&]+))(?:[?#&]|/(?!videos)|$)' % PornHubBaseIE._PORNHUB_HOST_RE
     _TESTS = [{
         'url': 'https://www.pornhub.com/model/zoe_ph',
         'playlist_mincount': 118,
@@ -549,6 +555,9 @@ class PornHubUserIE(PornHubPlaylistBaseIE):
         # Same as before, multi page
         'url': 'https://www.pornhubpremium.com/pornstar/lily-labeau',
         'only_matching': True,
+    }, {
+        'url': 'https://pornhubthbh7ap3u.onion/model/zoe_ph',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -624,7 +633,7 @@ class PornHubPagedPlaylistBaseIE(PornHubPlaylistBaseIE):
 
 
 class PornHubPagedVideoListIE(PornHubPagedPlaylistBaseIE):
-    _VALID_URL = r'https?://(?:[^/]+\.)?(?P<host>pornhub(?:premium)?\.(?:com|net|org))/(?P<id>(?:[^/]+/)*[^/?#&]+)'
+    _VALID_URL = r'https?://(?:[^/]+\.)?%s/(?P<id>(?:[^/]+/)*[^/?#&]+)' % PornHubBaseIE._PORNHUB_HOST_RE
     _TESTS = [{
         'url': 'https://www.pornhub.com/model/zoe_ph/videos',
         'only_matching': True,
@@ -729,6 +738,9 @@ class PornHubPagedVideoListIE(PornHubPagedPlaylistBaseIE):
     }, {
         'url': 'https://de.pornhub.com/playlist/4667351',
         'only_matching': True,
+    }, {
+        'url': 'https://pornhubthbh7ap3u.onion/model/zoe_ph/videos',
+        'only_matching': True,
     }]
 
     @classmethod
@@ -739,7 +751,7 @@ class PornHubPagedVideoListIE(PornHubPagedPlaylistBaseIE):
 
 
 class PornHubUserVideosUploadIE(PornHubPagedPlaylistBaseIE):
-    _VALID_URL = r'(?P<url>https?://(?:[^/]+\.)?(?P<host>pornhub(?:premium)?\.(?:com|net|org))/(?:(?:user|channel)s|model|pornstar)/(?P<id>[^/]+)/videos/upload)'
+    _VALID_URL = r'(?P<url>https?://(?:[^/]+\.)?%s/(?:(?:user|channel)s|model|pornstar)/(?P<id>[^/]+)/videos/upload)' % PornHubBaseIE._PORNHUB_HOST_RE
     _TESTS = [{
         'url': 'https://www.pornhub.com/pornstar/jenny-blighe/videos/upload',
         'info_dict': {
@@ -749,4 +761,7 @@ class PornHubUserVideosUploadIE(PornHubPagedPlaylistBaseIE):
     }, {
         'url': 'https://www.pornhub.com/model/zoe_ph/videos/upload',
         'only_matching': True,
+    }, {
+        'url': 'http://pornhubthbh7ap3u.onion/pornstar/jenny-blighe/videos/upload',
+        'only_matching': True,
     }]

From 379f52a4954013767219d25099cce9e0f9401961 Mon Sep 17 00:00:00 2001
From: Aleri Kaisattera <73682764+alerikaisattera@users.noreply.github.com>
Date: Mon, 21 Jun 2021 03:23:50 +0600
Subject: [PATCH 417/860] [liveleak] Remove extractor (closes #17625, closes
 #24222) (#29331)

---
 youtube_dl/extractor/extractors.py |   4 -
 youtube_dl/extractor/generic.py    |  31 -----
 youtube_dl/extractor/liveleak.py   | 191 -----------------------------
 3 files changed, 226 deletions(-)
 delete mode 100644 youtube_dl/extractor/liveleak.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 402e542ae..6e8fc3961 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -610,10 +610,6 @@ from .linkedin import (
 from .linuxacademy import LinuxAcademyIE
 from .litv import LiTVIE
 from .livejournal import LiveJournalIE
-from .liveleak import (
-    LiveLeakIE,
-    LiveLeakEmbedIE,
-)
 from .livestream import (
     LivestreamIE,
     LivestreamOriginalIE,
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 87594534f..a9c064105 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -84,7 +84,6 @@ from .jwplatform import JWPlatformIE
 from .digiteka import DigitekaIE
 from .arkena import ArkenaIE
 from .instagram import InstagramIE
-from .liveleak import LiveLeakIE
 from .threeqsdn import ThreeQSDNIE
 from .theplatform import ThePlatformIE
 from .kaltura import KalturaIE
@@ -1629,31 +1628,6 @@ class GenericIE(InfoExtractor):
                 'upload_date': '20160409',
             },
         },
-        # LiveLeak embed
-        {
-            'url': 'http://www.wykop.pl/link/3088787/',
-            'md5': '7619da8c820e835bef21a1efa2a0fc71',
-            'info_dict': {
-                'id': '874_1459135191',
-                'ext': 'mp4',
-                'title': 'Man shows poor quality of new apartment building',
-                'description': 'The wall is like a sand pile.',
-                'uploader': 'Lake8737',
-            },
-            'add_ie': [LiveLeakIE.ie_key()],
-        },
-        # Another LiveLeak embed pattern (#13336)
-        {
-            'url': 'https://milo.yiannopoulos.net/2017/06/concealed-carry-robbery/',
-            'info_dict': {
-                'id': '2eb_1496309988',
-                'ext': 'mp4',
-                'title': 'Thief robs place where everyone was armed',
-                'description': 'md5:694d73ee79e535953cf2488562288eee',
-                'uploader': 'brazilwtf',
-            },
-            'add_ie': [LiveLeakIE.ie_key()],
-        },
         # Duplicated embedded video URLs
         {
             'url': 'http://www.hudl.com/athlete/2538180/highlights/149298443',
@@ -3179,11 +3153,6 @@ class GenericIE(InfoExtractor):
             return self.url_result(
                 self._proto_relative_url(instagram_embed_url), InstagramIE.ie_key())
 
-        # Look for LiveLeak embeds
-        liveleak_urls = LiveLeakIE._extract_urls(webpage)
-        if liveleak_urls:
-            return self.playlist_from_matches(liveleak_urls, video_id, video_title)
-
         # Look for 3Q SDN embeds
         threeqsdn_url = ThreeQSDNIE._extract_url(webpage)
         if threeqsdn_url:
diff --git a/youtube_dl/extractor/liveleak.py b/youtube_dl/extractor/liveleak.py
deleted file mode 100644
index 4ac437c8b..000000000
--- a/youtube_dl/extractor/liveleak.py
+++ /dev/null
@@ -1,191 +0,0 @@
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import int_or_none
-
-
-class LiveLeakIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:\w+\.)?liveleak\.com/view\?.*?\b[it]=(?P<id>[\w_]+)'
-    _TESTS = [{
-        'url': 'http://www.liveleak.com/view?i=757_1364311680',
-        'md5': '0813c2430bea7a46bf13acf3406992f4',
-        'info_dict': {
-            'id': '757_1364311680',
-            'ext': 'mp4',
-            'description': 'extremely bad day for this guy..!',
-            'uploader': 'ljfriel2',
-            'title': 'Most unlucky car accident',
-            'thumbnail': r're:^https?://.*\.jpg$'
-        }
-    }, {
-        'url': 'http://www.liveleak.com/view?i=f93_1390833151',
-        'md5': 'd3f1367d14cc3c15bf24fbfbe04b9abf',
-        'info_dict': {
-            'id': 'f93_1390833151',
-            'ext': 'mp4',
-            'description': 'German Television Channel NDR does an exclusive interview with Edward Snowden.\r\nUploaded on LiveLeak cause German Television thinks the rest of the world isn\'t intereseted in Edward Snowden.',
-            'uploader': 'ARD_Stinkt',
-            'title': 'German Television does first Edward Snowden Interview (ENGLISH)',
-            'thumbnail': r're:^https?://.*\.jpg$'
-        }
-    }, {
-        # Prochan embed
-        'url': 'http://www.liveleak.com/view?i=4f7_1392687779',
-        'md5': '42c6d97d54f1db107958760788c5f48f',
-        'info_dict': {
-            'id': '4f7_1392687779',
-            'ext': 'mp4',
-            'description': "The guy with the cigarette seems amazingly nonchalant about the whole thing...  I really hope my friends' reactions would be a bit stronger.\r\n\r\nAction-go to 0:55.",
-            'uploader': 'CapObveus',
-            'title': 'Man is Fatally Struck by Reckless Car While Packing up a Moving Truck',
-            'age_limit': 18,
-        },
-        'skip': 'Video is dead',
-    }, {
-        # Covers https://github.com/ytdl-org/youtube-dl/pull/5983
-        # Multiple resolutions
-        'url': 'http://www.liveleak.com/view?i=801_1409392012',
-        'md5': 'c3a449dbaca5c0d1825caecd52a57d7b',
-        'info_dict': {
-            'id': '801_1409392012',
-            'ext': 'mp4',
-            'description': 'Happened on 27.7.2014. \r\nAt 0:53 you can see people still swimming at near beach.',
-            'uploader': 'bony333',
-            'title': 'Crazy Hungarian tourist films close call waterspout in Croatia',
-            'thumbnail': r're:^https?://.*\.jpg$'
-        }
-    }, {
-        # Covers https://github.com/ytdl-org/youtube-dl/pull/10664#issuecomment-247439521
-        'url': 'http://m.liveleak.com/view?i=763_1473349649',
-        'add_ie': ['Youtube'],
-        'info_dict': {
-            'id': '763_1473349649',
-            'ext': 'mp4',
-            'title': 'Reporters and public officials ignore epidemic of black on asian violence in Sacramento | Colin Flaherty',
-            'description': 'Colin being the warrior he is and showing the injustice Asians in Sacramento are being subjected to.',
-            'uploader': 'Ziz',
-            'upload_date': '20160908',
-            'uploader_id': 'UCEbta5E_jqlZmEJsriTEtnw'
-        },
-        'params': {
-            'skip_download': True,
-        },
-    }, {
-        'url': 'https://www.liveleak.com/view?i=677_1439397581',
-        'info_dict': {
-            'id': '677_1439397581',
-            'title': 'Fuel Depot in China Explosion caught on video',
-        },
-        'playlist_count': 3,
-    }, {
-        'url': 'https://www.liveleak.com/view?t=HvHi_1523016227',
-        'only_matching': True,
-    }, {
-        # No original video
-        'url': 'https://www.liveleak.com/view?t=C26ZZ_1558612804',
-        'only_matching': True,
-    }]
-
-    @staticmethod
-    def _extract_urls(webpage):
-        return re.findall(
-            r'<iframe[^>]+src="(https?://(?:\w+\.)?liveleak\.com/ll_embed\?[^"]*[ift]=[\w_]+[^"]+)"',
-            webpage)
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-
-        video_title = self._og_search_title(webpage).replace('LiveLeak.com -', '').strip()
-        video_description = self._og_search_description(webpage)
-        video_uploader = self._html_search_regex(
-            r'By:.*?(\w+)</a>', webpage, 'uploader', fatal=False)
-        age_limit = int_or_none(self._search_regex(
-            r'you confirm that you are ([0-9]+) years and over.',
-            webpage, 'age limit', default=None))
-        video_thumbnail = self._og_search_thumbnail(webpage)
-
-        entries = self._parse_html5_media_entries(url, webpage, video_id)
-        if not entries:
-            # Maybe an embed?
-            embed_url = self._search_regex(
-                r'<iframe[^>]+src="((?:https?:)?//(?:www\.)?(?:prochan|youtube)\.com/embed[^"]+)"',
-                webpage, 'embed URL')
-            return {
-                '_type': 'url_transparent',
-                'url': embed_url,
-                'id': video_id,
-                'title': video_title,
-                'description': video_description,
-                'uploader': video_uploader,
-                'age_limit': age_limit,
-            }
-
-        for idx, info_dict in enumerate(entries):
-            formats = []
-            for a_format in info_dict['formats']:
-                if not a_format.get('height'):
-                    a_format['height'] = int_or_none(self._search_regex(
-                        r'([0-9]+)p\.mp4', a_format['url'], 'height label',
-                        default=None))
-                formats.append(a_format)
-
-                # Removing '.*.mp4' gives the raw video, which is essentially
-                # the same video without the LiveLeak logo at the top (see
-                # https://github.com/ytdl-org/youtube-dl/pull/4768)
-                orig_url = re.sub(r'\.mp4\.[^.]+', '', a_format['url'])
-                if a_format['url'] != orig_url:
-                    format_id = a_format.get('format_id')
-                    format_id = 'original' + ('-' + format_id if format_id else '')
-                    if self._is_valid_url(orig_url, video_id, format_id):
-                        formats.append({
-                            'format_id': format_id,
-                            'url': orig_url,
-                            'preference': 1,
-                        })
-            self._sort_formats(formats)
-            info_dict['formats'] = formats
-
-            # Don't append entry ID for one-video pages to keep backward compatibility
-            if len(entries) > 1:
-                info_dict['id'] = '%s_%s' % (video_id, idx + 1)
-            else:
-                info_dict['id'] = video_id
-
-            info_dict.update({
-                'title': video_title,
-                'description': video_description,
-                'uploader': video_uploader,
-                'age_limit': age_limit,
-                'thumbnail': video_thumbnail,
-            })
-
-        return self.playlist_result(entries, video_id, video_title)
-
-
-class LiveLeakEmbedIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?liveleak\.com/ll_embed\?.*?\b(?P<kind>[ift])=(?P<id>[\w_]+)'
-
-    # See generic.py for actual test cases
-    _TESTS = [{
-        'url': 'https://www.liveleak.com/ll_embed?i=874_1459135191',
-        'only_matching': True,
-    }, {
-        'url': 'https://www.liveleak.com/ll_embed?f=ab065df993c1',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        kind, video_id = re.match(self._VALID_URL, url).groups()
-
-        if kind == 'f':
-            webpage = self._download_webpage(url, video_id)
-            liveleak_url = self._search_regex(
-                r'(?:logourl\s*:\s*|window\.open\()(?P<q1>[\'"])(?P<url>%s)(?P=q1)' % LiveLeakIE._VALID_URL,
-                webpage, 'LiveLeak URL', group='url')
-        else:
-            liveleak_url = 'http://www.liveleak.com/view?%s=%s' % (kind, video_id)
-
-        return self.url_result(liveleak_url, ie=LiveLeakIE.ie_key())

From 7fb9564420d43252c8f8c453d4dc54bf3ff9f8ee Mon Sep 17 00:00:00 2001
From: Remita Amine <remitamine@gmail.com>
Date: Mon, 28 Jun 2021 20:06:33 +0100
Subject: [PATCH 418/860] [periscope] pass referer to HLS requests(closes
 #29419)

---
 youtube_dl/extractor/periscope.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/periscope.py b/youtube_dl/extractor/periscope.py
index b15906390..b93a02b7d 100644
--- a/youtube_dl/extractor/periscope.py
+++ b/youtube_dl/extractor/periscope.py
@@ -12,6 +12,10 @@ from ..utils import (
 
 
 class PeriscopeBaseIE(InfoExtractor):
+    _M3U8_HEADERS = {
+        'Referer': 'https://www.periscope.tv/'
+    }
+
     def _call_api(self, method, query, item_id):
         return self._download_json(
             'https://api.periscope.tv/api/v2/%s' % method,
@@ -54,9 +58,11 @@ class PeriscopeBaseIE(InfoExtractor):
             m3u8_url, video_id, 'mp4',
             entry_protocol='m3u8_native'
             if state in ('ended', 'timed_out') else 'm3u8',
-            m3u8_id=format_id, fatal=fatal)
+            m3u8_id=format_id, fatal=fatal, headers=self._M3U8_HEADERS)
         if len(m3u8_formats) == 1:
             self._add_width_and_height(m3u8_formats[0], width, height)
+        for f in m3u8_formats:
+            f.setdefault('http_headers', {}).update(self._M3U8_HEADERS)
         return m3u8_formats
 
 

From a8035827177d6b59aca03bd717acb6a9bdd75ada Mon Sep 17 00:00:00 2001
From: bopol <bopol@e.email>
Date: Thu, 1 Jul 2021 08:53:22 +0200
Subject: [PATCH 419/860] [peertube] only call description endpoint if
 necessary (#29383)

---
 youtube_dl/extractor/peertube.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/peertube.py b/youtube_dl/extractor/peertube.py
index d9b13adc2..3af533925 100644
--- a/youtube_dl/extractor/peertube.py
+++ b/youtube_dl/extractor/peertube.py
@@ -569,15 +569,15 @@ class PeerTubeIE(InfoExtractor):
             formats.append(f)
         self._sort_formats(formats)
 
-        full_description = self._call_api(
-            host, video_id, 'description', note='Downloading description JSON',
-            fatal=False)
+        description = video.get('description')
+        if len(description) >= 250:
+            # description is shortened
+            full_description = self._call_api(
+                host, video_id, 'description', note='Downloading description JSON',
+                fatal=False)
 
-        description = None
-        if isinstance(full_description, dict):
-            description = str_or_none(full_description.get('description'))
-        if not description:
-            description = video.get('description')
+            if isinstance(full_description, dict):
+                description = str_or_none(full_description.get('description')) or description
 
         subtitles = self.extract_subtitles(host, video_id)
 

From b5242da7d24028f60cd23fd10f28fb635c7c7634 Mon Sep 17 00:00:00 2001
From: lanegramling <lanegramling@gmail.com>
Date: Thu, 16 Dec 2021 11:42:17 -0700
Subject: [PATCH 420/860] [youtube] Update signature function patterns (closes
 #30363) (#30366)

---
 youtube_dl/extractor/youtube.py | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index dc4bd4a77..62e58c13e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1323,10 +1323,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         funcname = self._search_regex(
             (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'\bm=(?P<sig>[a-zA-Z0-9$]{2})\(decodeURIComponent\(h\.s\)\)',
-             r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2})\(decodeURIComponent\(c\)\)',
-             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
-             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
+             r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
+             r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
+             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
+             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
              # Obsolete patterns
              r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',

From e41882335066ed03b1f4837e72fc0e83dfbe3525 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Dec 2021 01:43:16 +0700
Subject: [PATCH 421/860] [ChangeLog] Actualize [ci skip]

---
 ChangeLog | 25 +++++++++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/ChangeLog b/ChangeLog
index 680fffdf8..e530e6aea 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,3 +1,28 @@
+version <unreleased>
+
+Core
+* [postprocessor/ffmpeg] Show ffmpeg output on error (#22680, #29336)
+
+Extractors
+* [youtube] Update signature function patterns (#30363, #30366)
+* [peertube] Only call description endpoint if necessary (#29383)
+* [periscope] Pass referer to HLS requests (#29419)
+- [liveleak] Remove extractor (#17625, #24222, #29331)
++ [pornhub] Add support for pornhubthbh7ap3u.onion
+* [pornhub] Detect geo restriction
+* [pornhub] Dismiss tbr extracted from download URLs (#28927)
+* [curiositystream:collection] Extend _VALID_URL (#26326, #29117)
+* [youtube] Make get_video_info processing more robust (#29333)
+* [youtube] Workaround for get_video_info request (#29333)
+* [bilibili] Strip uploader name (#29202)
+* [youtube] Update invidious instance list (#29281)
+* [umg:de] Update GraphQL API URL (#29304)
+* [nrk] Switch psapi URL to https (#29344)
++ [egghead] Add support for app.egghead.io (#28404, #29303)
+* [appleconnect] Fix extraction (#29208)
++ [orf:tvthek] Add support for MPD formats (#28672, #29236)
+
+
 version 2021.06.06
 
 Extractors

From 5014bd67c22b421207b2650d4dc874b95b36dda1 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Sergey=20M=E2=80=A4?= <dstftw@gmail.com>
Date: Fri, 17 Dec 2021 01:49:07 +0700
Subject: [PATCH 422/860] release 2021.12.17

---
 .github/ISSUE_TEMPLATE/1_broken_site.md          | 6 +++---
 .github/ISSUE_TEMPLATE/2_site_support_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/3_site_feature_request.md | 4 ++--
 .github/ISSUE_TEMPLATE/4_bug_report.md           | 6 +++---
 .github/ISSUE_TEMPLATE/5_feature_request.md      | 4 ++--
 ChangeLog                                        | 2 +-
 docs/supportedsites.md                           | 2 --
 youtube_dl/version.py                            | 2 +-
 8 files changed, 14 insertions(+), 16 deletions(-)

diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md
index 4eb505231..e5405c235 100644
--- a/.github/ISSUE_TEMPLATE/1_broken_site.md
+++ b/.github/ISSUE_TEMPLATE/1_broken_site.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -26,7 +26,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support
-- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
+- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar issues including closed ones
@@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.06.06
+ [debug] youtube-dl version 2021.12.17
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md
index 9fed0b489..33b01ce7f 100644
--- a/.github/ISSUE_TEMPLATE/2_site_support_request.md
+++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md
@@ -19,7 +19,7 @@ labels: 'site-support-request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that site you are requesting is not dedicated to copyright infringement, see https://yt-dl.org/copyright-infringement. youtube-dl does not support such sites. In order for site support request to be accepted all provided example URLs should not violate any copyrights.
 - Search the bugtracker for similar site support requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a new site support request
-- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
+- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that none of provided URLs violate any copyrights
 - [ ] I've searched the bugtracker for similar site support requests including closed ones
diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
index 573e8ded0..285610cc7 100644
--- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md
@@ -18,13 +18,13 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar site feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a site feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
+- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
 - [ ] I've searched the bugtracker for similar site feature requests including closed ones
 
 
diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md
index c0031bf7a..af73525fb 100644
--- a/.github/ISSUE_TEMPLATE/4_bug_report.md
+++ b/.github/ISSUE_TEMPLATE/4_bug_report.md
@@ -18,7 +18,7 @@ title: ''
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Make sure that all provided video/audio/playlist URLs (if any) are alive and playable in a browser.
 - Make sure that all URLs and arguments with special characters are properly quoted or escaped as explained in http://yt-dl.org/escape.
 - Search the bugtracker for similar issues: http://yt-dl.org/search-issues. DO NOT post duplicates.
@@ -27,7 +27,7 @@ Carefully read and work through this check list in order to prevent the most com
 -->
 
 - [ ] I'm reporting a broken site support issue
-- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
+- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
 - [ ] I've checked that all provided URLs are alive and playable in a browser
 - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped
 - [ ] I've searched the bugtracker for similar bug reports including closed ones
@@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v <
  [debug] User config: []
  [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj']
  [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251
- [debug] youtube-dl version 2021.06.06
+ [debug] youtube-dl version 2021.12.17
  [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2
  [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4
  [debug] Proxy map: {}
diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md
index 1138ab2ca..42c878b83 100644
--- a/.github/ISSUE_TEMPLATE/5_feature_request.md
+++ b/.github/ISSUE_TEMPLATE/5_feature_request.md
@@ -19,13 +19,13 @@ labels: 'request'
 
 <!--
 Carefully read and work through this check list in order to prevent the most common mistakes and misuse of youtube-dl:
-- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.06.06. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
+- First of, make sure you are using the latest version of youtube-dl. Run `youtube-dl --version` and ensure your version is 2021.12.17. If it's not, see https://yt-dl.org/update on how to update. Issues with outdated version will be REJECTED.
 - Search the bugtracker for similar feature requests: http://yt-dl.org/search-issues. DO NOT post duplicates.
 - Finally, put x into all relevant boxes (like this [x])
 -->
 
 - [ ] I'm reporting a feature request
-- [ ] I've verified that I'm running youtube-dl version **2021.06.06**
+- [ ] I've verified that I'm running youtube-dl version **2021.12.17**
 - [ ] I've searched the bugtracker for similar feature requests including closed ones
 
 
diff --git a/ChangeLog b/ChangeLog
index e530e6aea..658864282 100644
--- a/ChangeLog
+++ b/ChangeLog
@@ -1,4 +1,4 @@
-version <unreleased>
+version 2021.12.17
 
 Core
 * [postprocessor/ffmpeg] Show ffmpeg output on error (#22680, #29336)
diff --git a/docs/supportedsites.md b/docs/supportedsites.md
index ed0d5e9d9..ae2a6b8b0 100644
--- a/docs/supportedsites.md
+++ b/docs/supportedsites.md
@@ -472,8 +472,6 @@
  - **LinuxAcademy**
  - **LiTV**
  - **LiveJournal**
- - **LiveLeak**
- - **LiveLeakEmbed**
  - **livestream**
  - **livestream:original**
  - **LnkGo**
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index 461dd87ca..b82fbc702 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.06.06'
+__version__ = '2021.12.17'

From ed99d68bdddfba0440dc81c105d5c0ea7cee7d1c Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 30 Jan 2022 00:41:47 +0530
Subject: [PATCH 423/860] Add back `YoutubeSearchURLIE`

---
 test/test_all_urls.py              |   6 +-
 youtube_dl/extractor/extractors.py |   2 +-
 youtube_dl/extractor/youtube.py    | 177 +++++++++++++++--------------
 3 files changed, 93 insertions(+), 92 deletions(-)

diff --git a/test/test_all_urls.py b/test/test_all_urls.py
index df6d81b5d..0e1328ede 100644
--- a/test/test_all_urls.py
+++ b/test/test_all_urls.py
@@ -66,9 +66,9 @@ class TestAllURLsMatching(unittest.TestCase):
         self.assertMatch('https://www.youtube.com/feed/watch_later', ['youtube:tab'])
         self.assertMatch('https://www.youtube.com/feed/subscriptions', ['youtube:tab'])
 
-    # def test_youtube_search_matching(self):
-    #     self.assertMatch('http://www.youtube.com/results?search_query=making+mustard', ['youtube:search_url'])
-    #     self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url'])
+    def test_youtube_search_matching(self):
+        self.assertMatch('http://www.youtube.com/results?search_query=making+mustard', ['youtube:search_url'])
+        self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url'])
 
     def test_youtube_extract(self):
         assertExtractId = lambda url, id: self.assertEqual(YoutubeIE.extract_id(url), id)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9b449937d..d403a2dbe 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1556,7 +1556,7 @@ from .youtube import (
     YoutubeRecommendedIE,
     YoutubeSearchDateIE,
     YoutubeSearchIE,
-    #YoutubeSearchURLIE,
+    YoutubeSearchURLIE,
     YoutubeSubscriptionsIE,
     YoutubeTruncatedIDIE,
     YoutubeTruncatedURLIE,
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 87bdc1677..578cfcf90 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -308,6 +308,77 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                 r'ytcfg\.set\s*\(\s*({.+?})\s*\)\s*;', webpage, 'ytcfg',
                 default='{}'), video_id, fatal=False)
 
+    def _search_results(self, query, params):
+        data = {
+            'context': {
+                'client': {
+                    'clientName': 'WEB',
+                    'clientVersion': '2.20201021.03.00',
+                }
+            },
+            'query': query,
+        }
+        if params:
+            data['params'] = params
+        for page_num in itertools.count(1):
+            search = self._download_json(
+                'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
+                video_id='query "%s"' % query,
+                note='Downloading page %s' % page_num,
+                errnote='Unable to download API page', fatal=False,
+                data=json.dumps(data).encode('utf8'),
+                headers={'content-type': 'application/json'})
+            if not search:
+                break
+            slr_contents = try_get(
+                search,
+                (lambda x: x['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents'],
+                 lambda x: x['onResponseReceivedCommands'][0]['appendContinuationItemsAction']['continuationItems']),
+                list)
+            if not slr_contents:
+                break
+            isr_contents = try_get(
+                slr_contents,
+                lambda x: x[0]['itemSectionRenderer']['contents'],
+                list)
+            if not isr_contents:
+                break
+            for content in isr_contents:
+                if not isinstance(content, dict):
+                    continue
+                video = content.get('videoRenderer')
+                if not isinstance(video, dict):
+                    continue
+                video_id = video.get('videoId')
+                if not video_id:
+                    continue
+                title = try_get(video, lambda x: x['title']['runs'][0]['text'], compat_str)
+                description = try_get(video, lambda x: x['descriptionSnippet']['runs'][0]['text'], compat_str)
+                duration = parse_duration(try_get(video, lambda x: x['lengthText']['simpleText'], compat_str))
+                view_count_text = try_get(video, lambda x: x['viewCountText']['simpleText'], compat_str) or ''
+                view_count = int_or_none(self._search_regex(
+                    r'^(\d+)', re.sub(r'\s', '', view_count_text),
+                    'view count', default=None))
+                uploader = try_get(video, lambda x: x['ownerText']['runs'][0]['text'], compat_str)
+                yield {
+                    '_type': 'url_transparent',
+                    'ie_key': YoutubeIE.ie_key(),
+                    'id': video_id,
+                    'url': video_id,
+                    'title': title,
+                    'description': description,
+                    'duration': duration,
+                    'view_count': view_count,
+                    'uploader': uploader,
+                }
+            token = try_get(
+                slr_contents,
+                lambda x: x[1]['continuationItemRenderer']['continuationEndpoint']['continuationCommand']['token'],
+                compat_str)
+            if not token:
+                break
+            data['continuation'] = token
+
 
 class YoutubeIE(YoutubeBaseInfoExtractor):
     IE_DESC = 'YouTube.com'
@@ -2454,7 +2525,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                         (?:
                             (?:channel|c|user|feed)/|
                             (?:playlist|watch)\?.*?\blist=|
-                            (?!(?:watch|embed|v|e)\b)
+                            (?!(?:watch|embed|v|e|results)\b)
                         )
                         (?P<id>[^/?\#&]+)
                     '''
@@ -3379,88 +3450,18 @@ class YoutubeFavouritesIE(YoutubeBaseInfoExtractor):
 
 class YoutubeSearchIE(SearchInfoExtractor, YoutubeBaseInfoExtractor):
     IE_DESC = 'YouTube.com searches'
-    # there doesn't appear to be a real limit, for example if you search for
-    # 'python' you get more than 8.000.000 results
-    _MAX_RESULTS = float('inf')
     IE_NAME = 'youtube:search'
     _SEARCH_KEY = 'ytsearch'
-    _SEARCH_PARAMS = None
+    _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
     _TESTS = []
 
     def _entries(self, query, n):
-        data = {
-            'context': {
-                'client': {
-                    'clientName': 'WEB',
-                    'clientVersion': '2.20201021.03.00',
-                }
-            },
-            'query': query,
-        }
-        if self._SEARCH_PARAMS:
-            data['params'] = self._SEARCH_PARAMS
         total = 0
-        for page_num in itertools.count(1):
-            search = self._download_json(
-                'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
-                video_id='query "%s"' % query,
-                note='Downloading page %s' % page_num,
-                errnote='Unable to download API page', fatal=False,
-                data=json.dumps(data).encode('utf8'),
-                headers={'content-type': 'application/json'})
-            if not search:
-                break
-            slr_contents = try_get(
-                search,
-                (lambda x: x['contents']['twoColumnSearchResultsRenderer']['primaryContents']['sectionListRenderer']['contents'],
-                 lambda x: x['onResponseReceivedCommands'][0]['appendContinuationItemsAction']['continuationItems']),
-                list)
-            if not slr_contents:
-                break
-            isr_contents = try_get(
-                slr_contents,
-                lambda x: x[0]['itemSectionRenderer']['contents'],
-                list)
-            if not isr_contents:
-                break
-            for content in isr_contents:
-                if not isinstance(content, dict):
-                    continue
-                video = content.get('videoRenderer')
-                if not isinstance(video, dict):
-                    continue
-                video_id = video.get('videoId')
-                if not video_id:
-                    continue
-                title = try_get(video, lambda x: x['title']['runs'][0]['text'], compat_str)
-                description = try_get(video, lambda x: x['descriptionSnippet']['runs'][0]['text'], compat_str)
-                duration = parse_duration(try_get(video, lambda x: x['lengthText']['simpleText'], compat_str))
-                view_count_text = try_get(video, lambda x: x['viewCountText']['simpleText'], compat_str) or ''
-                view_count = int_or_none(self._search_regex(
-                    r'^(\d+)', re.sub(r'\s', '', view_count_text),
-                    'view count', default=None))
-                uploader = try_get(video, lambda x: x['ownerText']['runs'][0]['text'], compat_str)
-                total += 1
-                yield {
-                    '_type': 'url_transparent',
-                    'ie_key': YoutubeIE.ie_key(),
-                    'id': video_id,
-                    'url': video_id,
-                    'title': title,
-                    'description': description,
-                    'duration': duration,
-                    'view_count': view_count,
-                    'uploader': uploader,
-                }
-                if total == n:
-                    return
-            token = try_get(
-                slr_contents,
-                lambda x: x[1]['continuationItemRenderer']['continuationEndpoint']['continuationCommand']['token'],
-                compat_str)
-            if not token:
-                break
-            data['continuation'] = token
+        for entry in self._search_results(query, self._SEARCH_PARAMS):
+            yield entry
+            total += 1
+            if total >= n:
+                return
 
     def _get_n_results(self, query, n):
         """Get a specified number of results for a query"""
@@ -3471,18 +3472,19 @@ class YoutubeSearchDateIE(YoutubeSearchIE):
     IE_NAME = YoutubeSearchIE.IE_NAME + ':date'
     _SEARCH_KEY = 'ytsearchdate'
     IE_DESC = 'YouTube.com searches, newest videos first'
-    _SEARCH_PARAMS = 'CAI%3D'
+    _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
+    _TESTS = []
 
 
-r"""
-class YoutubeSearchURLIE(YoutubeSearchIE):
-    IE_DESC = 'YouTube.com search URLs'
-    IE_NAME = 'youtube:search_url'
-    _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?P<query>[^&]+)(?:[&]|$)'
+class YoutubeSearchURLIE(YoutubeBaseInfoExtractor):
+    IE_DESC = 'YouTube search URLs with sorting and filter support'
+    IE_NAME = YoutubeSearchIE.IE_NAME + '_url'
+    _VALID_URL = r'https?://(?:www\.)?youtube\.com/results\?(.*?&)?(?:search_query|q)=(?:[^&]+)(?:[&]|$)'
     _TESTS = [{
         'url': 'https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video',
         'playlist_mincount': 5,
         'info_dict': {
+            'id': 'youtube-dl test video',
             'title': 'youtube-dl test video',
         }
     }, {
@@ -3491,11 +3493,10 @@ class YoutubeSearchURLIE(YoutubeSearchIE):
     }]
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        query = compat_urllib_parse_unquote_plus(mobj.group('query'))
-        webpage = self._download_webpage(url, query)
-        return self.playlist_result(self._process_page(webpage), playlist_title=query)
-"""
+        qs = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
+        query = (qs.get('search_query') or qs.get('q'))[0]
+        params = qs.get('sp', ('',))[0]
+        return self.playlist_result(self._search_results(query, params), query, query)
 
 
 class YoutubeFeedsInfoExtractor(YoutubeTabIE):

From bfe72723d8318f8bfcb35dee69a40758df5fa3c0 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 30 Jan 2022 00:49:55 +0530
Subject: [PATCH 424/860] Use `itertools.islice`

---
 youtube_dl/extractor/youtube.py | 11 ++---------
 1 file changed, 2 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 578cfcf90..017837e10 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -3455,17 +3455,10 @@ class YoutubeSearchIE(SearchInfoExtractor, YoutubeBaseInfoExtractor):
     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
     _TESTS = []
 
-    def _entries(self, query, n):
-        total = 0
-        for entry in self._search_results(query, self._SEARCH_PARAMS):
-            yield entry
-            total += 1
-            if total >= n:
-                return
-
     def _get_n_results(self, query, n):
         """Get a specified number of results for a query"""
-        return self.playlist_result(self._entries(query, n), query)
+        entries = itertools.islice(self._search_results(query, self._SEARCH_PARAMS), 0, None if n == float('inf') else n)
+        return self.playlist_result(entries, query, query)
 
 
 class YoutubeSearchDateIE(YoutubeSearchIE):

From 2c4cb134a90b49a4d44965b57ff43cfd45ec2d69 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 30 Jan 2022 00:54:22 +0530
Subject: [PATCH 425/860] Fix max_results

---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 017837e10..bbd3e80d8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -3453,6 +3453,7 @@ class YoutubeSearchIE(SearchInfoExtractor, YoutubeBaseInfoExtractor):
     IE_NAME = 'youtube:search'
     _SEARCH_KEY = 'ytsearch'
     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
+    _MAX_RESULTS = float('inf')
     _TESTS = []
 
     def _get_n_results(self, query, n):

From 57044eacebc6f2f3cd83c345e1b6e659a22e4773 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Thu, 28 Oct 2021 15:55:38 +0100
Subject: [PATCH 426/860] Fix test_youtube_playlist_noplaylist

---
 test/test_youtube_lists.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index cf2fdf14f..72820972e 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# -*- coding: utf-8 -*-
 from __future__ import unicode_literals
 
 # Allow direct execution
@@ -9,11 +10,10 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from test.helper import FakeYDL
 
-
 from youtube_dl.extractor import (
+    YoutubeIE,
     YoutubePlaylistIE,
     YoutubeTabIE,
-    YoutubeIE,
 )
 
 
@@ -25,9 +25,11 @@ class TestYoutubeLists(unittest.TestCase):
     def test_youtube_playlist_noplaylist(self):
         dl = FakeYDL()
         dl.params['noplaylist'] = True
+        dl.params['format'] = 'best'
         ie = YoutubePlaylistIE(dl)
         result = ie.extract('https://www.youtube.com/watch?v=FXxLjLQi3Fg&list=PLwiyx1dc3P2JR9N8gQaQN_BCvlSlap7re')
         self.assertEqual(result['_type'], 'url')
+        result = dl.extract_info(result['url'], download=False, ie_key=result.get('ie_key'), process=False)
         self.assertEqual(YoutubeIE().extract_id(result['url']), 'FXxLjLQi3Fg')
 
     def test_youtube_course(self):

From 46e0a729b2d4503d8d49433fdddfce726d08261e Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Thu, 28 Oct 2021 15:57:10 +0100
Subject: [PATCH 427/860] Remove obsolete test_youtube_course

---
 test/test_youtube_lists.py | 10 ----------
 1 file changed, 10 deletions(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index 72820972e..e1636a1a6 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -32,16 +32,6 @@ class TestYoutubeLists(unittest.TestCase):
         result = dl.extract_info(result['url'], download=False, ie_key=result.get('ie_key'), process=False)
         self.assertEqual(YoutubeIE().extract_id(result['url']), 'FXxLjLQi3Fg')
 
-    def test_youtube_course(self):
-        dl = FakeYDL()
-        ie = YoutubePlaylistIE(dl)
-        # TODO find a > 100 (paginating?) videos course
-        result = ie.extract('https://www.youtube.com/course?list=ECUl4u3cNGP61MdtwGTqZA0MreSaDybji8')
-        entries = list(result['entries'])
-        self.assertEqual(YoutubeIE().extract_id(entries[0]['url']), 'j9WZyLZCBzs')
-        self.assertEqual(len(entries), 25)
-        self.assertEqual(YoutubeIE().extract_id(entries[-1]['url']), 'rYefUsYuEp0')
-
     def test_youtube_mix(self):
         dl = FakeYDL()
         ie = YoutubePlaylistIE(dl)

From 2c2c2bd348b7dce0aad55a6fc37a18c6f9a000e3 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Fri, 29 Oct 2021 03:03:00 +0100
Subject: [PATCH 428/860] Fix test_youtube_mix

---
 test/test_youtube_lists.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index e1636a1a6..fae8a950a 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -34,12 +34,14 @@ class TestYoutubeLists(unittest.TestCase):
 
     def test_youtube_mix(self):
         dl = FakeYDL()
-        ie = YoutubePlaylistIE(dl)
-        result = ie.extract('https://www.youtube.com/watch?v=W01L70IGBgE&index=2&list=RDOQpdSVF_k_w')
-        entries = result['entries']
+        dl.params['format'] = 'best'
+        ie = YoutubeTabIE(dl)
+        result = dl.extract_info('https://www.youtube.com/watch?v=uVJ0Il5WvbE&list=PLhQjrBD2T381k8ul4WQ8SQ165XqY149WW',
+                                 download=False, ie_key=ie.ie_key(), process=True)
+        entries = (result or {}).get('entries', [{'id': 'not_found', }])
         self.assertTrue(len(entries) >= 50)
         original_video = entries[0]
-        self.assertEqual(original_video['id'], 'OQpdSVF_k_w')
+        self.assertEqual(original_video['id'], 'uVJ0Il5WvbE')
 
     def test_youtube_toptracks(self):
         print('Skipping: The playlist page gives error 500')

From d76d59d99d05fba94963690a039d38373dddc658 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Fri, 29 Oct 2021 03:10:35 +0100
Subject: [PATCH 429/860] Remove obsolete non-working test_youtube_toptracks

---
 test/test_youtube_lists.py | 9 ---------
 1 file changed, 9 deletions(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index fae8a950a..69c5d52eb 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -43,15 +43,6 @@ class TestYoutubeLists(unittest.TestCase):
         original_video = entries[0]
         self.assertEqual(original_video['id'], 'uVJ0Il5WvbE')
 
-    def test_youtube_toptracks(self):
-        print('Skipping: The playlist page gives error 500')
-        return
-        dl = FakeYDL()
-        ie = YoutubePlaylistIE(dl)
-        result = ie.extract('https://www.youtube.com/playlist?list=MCUS')
-        entries = result['entries']
-        self.assertEqual(len(entries), 100)
-
     def test_youtube_flat_playlist_extraction(self):
         dl = FakeYDL()
         dl.params['extract_flat'] = True

From 39ca35e7651048c2adf558f1d6db2df0de4554f5 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Mon, 1 Nov 2021 04:44:57 +0000
Subject: [PATCH 430/860] Fix test_youtube_flat_playlist_extraction

---
 test/test_youtube_lists.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index 69c5d52eb..07a6b6d06 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -52,7 +52,7 @@ class TestYoutubeLists(unittest.TestCase):
         entries = list(result['entries'])
         self.assertTrue(len(entries) == 1)
         video = entries[0]
-        self.assertEqual(video['_type'], 'url_transparent')
+        self.assertEqual(video['_type'], 'url')
         self.assertEqual(video['ie_key'], 'Youtube')
         self.assertEqual(video['id'], 'BaW_jenozKc')
         self.assertEqual(video['url'], 'BaW_jenozKc')

From 5f5de51a499f732a6e687f32037e130cbdc50c8f Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Mon, 1 Nov 2021 13:34:29 +0000
Subject: [PATCH 431/860] Add compat_map/filter and use the former

---
 youtube_dl/compat.py            | 21 +++++++++++++++++++++
 youtube_dl/extractor/youtube.py |  1 +
 2 files changed, 22 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9e45c454b..29e0d3a02 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2962,6 +2962,25 @@ else:
         compat_Struct = struct.Struct
 
 
+# compat_map/filter() returning an iterator, supposedly the
+# same versioning as for zip below
+try:
+    from future_builtins import map as compat_map
+except ImportError:
+    try:
+        from itertools import imap as compat_map
+    except ImportError:
+        compat_map = map
+
+try:
+    from future_builtins import filter as compat_filter
+except ImportError:
+    try:
+        from itertools import ifilter as compat_filter
+    except ImportError:
+        compat_filter = filter
+
+
 try:
     from future_builtins import zip as compat_zip
 except ImportError:  # not 2.6+ or is 3.x
@@ -3015,6 +3034,7 @@ __all__ = [
     'compat_etree_fromstring',
     'compat_etree_register_namespace',
     'compat_expanduser',
+    'compat_filter',
     'compat_get_terminal_size',
     'compat_getenv',
     'compat_getpass',
@@ -3026,6 +3046,7 @@ __all__ = [
     'compat_integer_types',
     'compat_itertools_count',
     'compat_kwargs',
+    'compat_map',
     'compat_numeric_types',
     'compat_ord',
     'compat_os_name',
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 62e58c13e..da410f8f0 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -13,6 +13,7 @@ from .common import InfoExtractor, SearchInfoExtractor
 from ..compat import (
     compat_chr,
     compat_HTTPError,
+    compat_map as map,
     compat_parse_qs,
     compat_str,
     compat_urllib_parse_unquote_plus,

From 96f87aaa3b34d80bc72097a7475d8093849091fc Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Tue, 2 Nov 2021 11:18:39 +0000
Subject: [PATCH 432/860] Back-port JS interpreter upgrade from yt-dlp PR #1437

---
 test/test_jsinterp.py  |  51 +++++
 youtube_dl/compat.py   |   5 +
 youtube_dl/jsinterp.py | 496 ++++++++++++++++++++++++++++++++---------
 3 files changed, 449 insertions(+), 103 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index c24b8ca74..4d05ea610 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -112,6 +112,57 @@ class TestJSInterpreter(unittest.TestCase):
         ''')
         self.assertEqual(jsi.call_function('z'), 5)
 
+    def test_for_loop(self):
+        # function x() { a=0; for (i=0; i-10; i++) {a++} a }
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i = i + 1) {a++} a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+    def test_switch(self):
+        jsi = JSInterpreter('''
+        function x(f) { switch(f){
+            case 1:f+=1;
+            case 2:f+=2;
+            case 3:f+=3;break;
+            case 4:f+=4;
+            default:f=0;
+        } return f }
+        ''')
+        self.assertEqual(jsi.call_function('x', 1), 7)
+        self.assertEqual(jsi.call_function('x', 3), 6)
+        self.assertEqual(jsi.call_function('x', 5), 0)
+
+    def test_try(self):
+        jsi = JSInterpreter('''
+        function x() { try{return 10} catch(e){return 5} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+    def test_for_loop_continue(self):
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i++) { continue; a++ } a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+    def test_for_loop_break(self):
+        jsi = JSInterpreter('''
+        function x() { a=0; for (i=0; i-10; i++) { break; a++ } a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+    def test_literal_list(self):
+        jsi = JSInterpreter('''
+        function x() { [1, 2, "asdf", [5, 6, 7]][3] }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [5, 6, 7])
+
+    def test_comma(self):
+        jsi = JSInterpreter('''
+        function x() { a=5; a -= 1, a+=3; return a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 7)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 29e0d3a02..2004a405a 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -21,6 +21,10 @@ import subprocess
 import sys
 import xml.etree.ElementTree
 
+try:
+    import collections.abc as compat_collections_abc
+except ImportError:
+    import collections as compat_collections_abc
 
 try:
     import urllib.request as compat_urllib_request
@@ -3025,6 +3029,7 @@ __all__ = [
     'compat_b64decode',
     'compat_basestring',
     'compat_chr',
+    'compat_collections_abc',
     'compat_cookiejar',
     'compat_cookiejar_Cookie',
     'compat_cookies',
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 7bda59610..061e92c2a 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -8,6 +8,15 @@ from .utils import (
     ExtractorError,
     remove_quotes,
 )
+from .compat import (
+    compat_collections_abc
+)
+MutableMapping = compat_collections_abc.MutableMapping
+
+
+class Nonlocal:
+    pass
+
 
 _OPERATORS = [
     ('|', operator.or_),
@@ -22,11 +31,55 @@ _OPERATORS = [
     ('*', operator.mul),
 ]
 _ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS]
-_ASSIGN_OPERATORS.append(('=', lambda cur, right: right))
+_ASSIGN_OPERATORS.append(('=', (lambda cur, right: right)))
 
 _NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*'
 
 
+class JS_Break(ExtractorError):
+    def __init__(self):
+        ExtractorError.__init__(self, 'Invalid break')
+
+
+class JS_Continue(ExtractorError):
+    def __init__(self):
+        ExtractorError.__init__(self, 'Invalid continue')
+
+
+class LocalNameSpace(MutableMapping):
+    def __init__(self, *stack):
+        self.stack = tuple(stack)
+
+    def __getitem__(self, key):
+        for scope in self.stack:
+            if key in scope:
+                return scope[key]
+        raise KeyError(key)
+
+    def __setitem__(self, key, value):
+        for scope in self.stack:
+            if key in scope:
+                scope[key] = value
+                break
+        else:
+            self.stack[0][key] = value
+        return value
+
+    def __delitem__(self, key):
+        raise NotImplementedError('Deleting is not supported')
+
+    def __iter__(self):
+        for scope in self.stack:
+            for scope_item in iter(scope):
+                yield scope_item
+
+    def __len__(self, key):
+        return len(iter(self))
+
+    def __repr__(self):
+        return 'LocalNameSpace%s' % (self.stack, )
+
+
 class JSInterpreter(object):
     def __init__(self, code, objects=None):
         if objects is None:
@@ -34,11 +87,58 @@ class JSInterpreter(object):
         self.code = code
         self._functions = {}
         self._objects = objects
+        self.__named_object_counter = 0
+
+    def _named_object(self, namespace, obj):
+        self.__named_object_counter += 1
+        name = '__youtube_dl_jsinterp_obj%s' % (self.__named_object_counter, )
+        namespace[name] = obj
+        return name
+
+    @staticmethod
+    def _separate(expr, delim=',', max_split=None):
+        if not expr:
+            return
+        parens = {'(': 0, '{': 0, '[': 0, ']': 0, '}': 0, ')': 0}
+        start, splits, pos, max_pos = 0, 0, 0, len(delim) - 1
+        for idx, char in enumerate(expr):
+            if char in parens:
+                parens[char] += 1
+            is_in_parens = (parens['['] - parens[']']
+                            or parens['('] - parens[')']
+                            or parens['{'] - parens['}'])
+            if char == delim[pos] and not is_in_parens:
+                if pos == max_pos:
+                    pos = 0
+                    yield expr[start: idx - max_pos]
+                    start = idx + 1
+                    splits += 1
+                    if max_split and splits >= max_split:
+                        break
+                else:
+                    pos += 1
+            else:
+                pos = 0
+        yield expr[start:]
+
+    @staticmethod
+    def _separate_at_paren(expr, delim):
+        separated = list(JSInterpreter._separate(expr, delim, 1))
+        if len(separated) < 2:
+            raise ExtractorError('No terminating paren {0} in {1}'.format(delim, expr))
+        return separated[0][1:].strip(), separated[1].strip()
 
     def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if allow_recursion < 0:
             raise ExtractorError('Recursion limit reached')
 
+        sub_statements = list(self._separate(stmt, ';'))
+        stmt = (sub_statements or ['']).pop()
+        for sub_stmt in sub_statements:
+            ret, should_abort = self.interpret_statement(sub_stmt, local_vars, allow_recursion - 1)
+            if should_abort:
+                return ret
+
         should_abort = False
         stmt = stmt.lstrip()
         stmt_m = re.match(r'var\s', stmt)
@@ -61,25 +161,119 @@ class JSInterpreter(object):
         if expr == '':  # Empty expression
             return None
 
-        if expr.startswith('('):
-            parens_count = 0
-            for m in re.finditer(r'[()]', expr):
-                if m.group(0) == '(':
-                    parens_count += 1
-                else:
-                    parens_count -= 1
-                    if parens_count == 0:
-                        sub_expr = expr[1:m.start()]
-                        sub_result = self.interpret_expression(
-                            sub_expr, local_vars, allow_recursion)
-                        remaining_expr = expr[m.end():].strip()
-                        if not remaining_expr:
-                            return sub_result
-                        else:
-                            expr = json.dumps(sub_result) + remaining_expr
-                        break
+        if expr.startswith('{'):
+            inner, outer = self._separate_at_paren(expr, '}')
+            inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion - 1)
+            if not outer or should_abort:
+                return inner
             else:
-                raise ExtractorError('Premature end of parens in %r' % expr)
+                expr = json.dumps(inner) + outer
+
+        if expr.startswith('('):
+            inner, outer = self._separate_at_paren(expr, ')')
+            inner = self.interpret_expression(inner, local_vars, allow_recursion)
+            if not outer:
+                return inner
+            else:
+                expr = json.dumps(inner) + outer
+
+        if expr.startswith('['):
+            inner, outer = self._separate_at_paren(expr, ']')
+            name = self._named_object(local_vars, [
+                self.interpret_expression(item, local_vars, allow_recursion)
+                for item in self._separate(inner)])
+            expr = name + outer
+
+        m = re.match(r'try\s*', expr)
+        if m:
+            if expr[m.end()] == '{':
+                try_expr, expr = self._separate_at_paren(expr[m.end():], '}')
+            else:
+                try_expr, expr = expr[m.end() - 1:], ''
+            ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion - 1)
+            if should_abort:
+                return ret
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+
+        m = re.match(r'(?:(?P<catch>catch)|(?P<for>for)|(?P<switch>switch))\s*\(', expr)
+        md = m.groupdict() if m else {}
+        if md.get('catch'):
+            # We ignore the catch block
+            _, expr = self._separate_at_paren(expr, '}')
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+
+        elif md.get('for'):
+            def raise_constructor_error(c):
+                raise ExtractorError(
+                    'Premature return in the initialization of a for loop in {0!r}'.format(c))
+
+            constructor, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
+            if remaining.startswith('{'):
+                body, expr = self._separate_at_paren(remaining, '}')
+            else:
+                m = re.match(r'switch\s*\(', remaining)  # FIXME
+                if m:
+                    switch_val, remaining = self._separate_at_paren(remaining[m.end() - 1:], ')')
+                    body, expr = self._separate_at_paren(remaining, '}')
+                    body = 'switch(%s){%s}' % (switch_val, body)
+                else:
+                    body, expr = remaining, ''
+            start, cndn, increment = self._separate(constructor, ';')
+            if self.interpret_statement(start, local_vars, allow_recursion - 1)[1]:
+                raise_constructor_error(constructor)
+            while True:
+                if not self.interpret_expression(cndn, local_vars, allow_recursion):
+                    break
+                try:
+                    ret, should_abort = self.interpret_statement(body, local_vars, allow_recursion - 1)
+                    if should_abort:
+                        return ret
+                except JS_Break:
+                    break
+                except JS_Continue:
+                    pass
+                if self.interpret_statement(increment, local_vars, allow_recursion - 1)[1]:
+                    raise_constructor_error(constructor)
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+
+        elif md.get('switch'):
+            switch_val, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
+            switch_val = self.interpret_expression(switch_val, local_vars, allow_recursion)
+            body, expr = self._separate_at_paren(remaining, '}')
+            body, default = body.split('default:') if 'default:' in body else (body, None)
+            items = body.split('case ')[1:]
+            if default:
+                items.append('default:%s' % (default, ))
+            matched = False
+            for item in items:
+                case, stmt = [i.strip() for i in self._separate(item, ':', 1)]
+                matched = matched or case == 'default' or switch_val == self.interpret_expression(case, local_vars, allow_recursion)
+                if matched:
+                    try:
+                        ret, should_abort = self.interpret_statement(stmt, local_vars, allow_recursion - 1)
+                        if should_abort:
+                            return ret
+                    except JS_Break:
+                        break
+            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+
+        # Comma separated statements
+        sub_expressions = list(self._separate(expr))
+        expr = sub_expressions.pop().strip() if sub_expressions else ''
+        for sub_expr in sub_expressions:
+            self.interpret_expression(sub_expr, local_vars, allow_recursion)
+
+        for m in re.finditer(r'''(?x)
+                (?P<pre_sign>\+\+|--)(?P<var1>%(_NAME_RE)s)|
+                (?P<var2>%(_NAME_RE)s)(?P<post_sign>\+\+|--)''' % globals(), expr):
+            var = m.group('var1') or m.group('var2')
+            start, end = m.span()
+            sign = m.group('pre_sign') or m.group('post_sign')
+            ret = local_vars[var]
+            local_vars[var] += 1 if sign[0] == '+' else -1
+            if m.group('pre_sign'):
+                ret = local_vars[var]
+            expr = expr[:start] + json.dumps(ret) + expr[end:]
 
         for op, opfunc in _ASSIGN_OPERATORS:
             m = re.match(r'''(?x)
@@ -88,14 +282,13 @@ class JSInterpreter(object):
                 (?P<expr>.*)$''' % (_NAME_RE, re.escape(op)), expr)
             if not m:
                 continue
-            right_val = self.interpret_expression(
-                m.group('expr'), local_vars, allow_recursion - 1)
+            right_val = self.interpret_expression(m.group('expr'), local_vars, allow_recursion)
 
             if m.groupdict().get('index'):
                 lvar = local_vars[m.group('out')]
-                idx = self.interpret_expression(
-                    m.group('index'), local_vars, allow_recursion)
-                assert isinstance(idx, int)
+                idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
+                if not isinstance(idx, int):
+                    raise ExtractorError('List indices must be integers: %s' % (idx, ))
                 cur = lvar[idx]
                 val = opfunc(cur, right_val)
                 lvar[idx] = val
@@ -109,8 +302,13 @@ class JSInterpreter(object):
         if expr.isdigit():
             return int(expr)
 
+        if expr == 'break':
+            raise JS_Break()
+        elif expr == 'continue':
+            raise JS_Continue()
+
         var_m = re.match(
-            r'(?!if|return|true|false)(?P<name>%s)$' % _NAME_RE,
+            r'(?!if|return|true|false|null)(?P<name>%s)$' % _NAME_RE,
             expr)
         if var_m:
             return local_vars[var_m.group('name')]
@@ -124,91 +322,161 @@ class JSInterpreter(object):
             r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
         if m:
             val = local_vars[m.group('in')]
-            idx = self.interpret_expression(
-                m.group('idx'), local_vars, allow_recursion - 1)
+            idx = self.interpret_expression(m.group('idx'), local_vars, allow_recursion)
             return val[idx]
 
+        def raise_expr_error(where, op, exp):
+            raise ExtractorError('Premature {0} return of {1} in {2!r}'.format(where, op, exp))
+
+        for op, opfunc in _OPERATORS:
+            separated = list(self._separate(expr, op))
+            if len(separated) < 2:
+                continue
+            right_val = separated.pop()
+            left_val = op.join(separated)
+            left_val, should_abort = self.interpret_statement(
+                left_val, local_vars, allow_recursion - 1)
+            if should_abort:
+                raise_expr_error('left-side', op, expr)
+            right_val, should_abort = self.interpret_statement(
+                right_val, local_vars, allow_recursion - 1)
+            if should_abort:
+                raise_expr_error('right-side', op, expr)
+            return opfunc(left_val or 0, right_val)
+
         m = re.match(
-            r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*(?:\(+(?P<args>[^()]*)\))?$' % _NAME_RE,
+            r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*' % _NAME_RE,
             expr)
         if m:
             variable = m.group('var')
-            member = remove_quotes(m.group('member') or m.group('member2'))
-            arg_str = m.group('args')
+            nl = Nonlocal()
 
-            if variable in local_vars:
-                obj = local_vars[variable]
+            nl.member = remove_quotes(m.group('member') or m.group('member2'))
+            arg_str = expr[m.end():]
+            if arg_str.startswith('('):
+                arg_str, remaining = self._separate_at_paren(arg_str, ')')
             else:
-                if variable not in self._objects:
-                    self._objects[variable] = self.extract_object(variable)
-                obj = self._objects[variable]
+                arg_str, remaining = None, arg_str
 
-            if arg_str is None:
-                # Member access
-                if member == 'length':
-                    return len(obj)
-                return obj[member]
+            def assertion(cndn, msg):
+                """ assert, but without risk of getting optimized out """
+                if not cndn:
+                    raise ExtractorError('{0} {1}: {2}'.format(nl.member, msg, expr))
 
-            assert expr.endswith(')')
-            # Function call
-            if arg_str == '':
-                argvals = tuple()
-            else:
-                argvals = tuple([
+            def eval_method():
+                # nonlocal member
+                member = nl.member
+                if variable == 'String':
+                    obj = str
+                elif variable in local_vars:
+                    obj = local_vars[variable]
+                else:
+                    if variable not in self._objects:
+                        self._objects[variable] = self.extract_object(variable)
+                    obj = self._objects[variable]
+
+                if arg_str is None:
+                    # Member access
+                    if member == 'length':
+                        return len(obj)
+                    return obj[member]
+
+                # Function call
+                argvals = [
                     self.interpret_expression(v, local_vars, allow_recursion)
-                    for v in arg_str.split(',')])
+                    for v in self._separate(arg_str)]
 
-            if member == 'split':
-                assert argvals == ('',)
-                return list(obj)
-            if member == 'join':
-                assert len(argvals) == 1
-                return argvals[0].join(obj)
-            if member == 'reverse':
-                assert len(argvals) == 0
-                obj.reverse()
-                return obj
-            if member == 'slice':
-                assert len(argvals) == 1
-                return obj[argvals[0]:]
-            if member == 'splice':
-                assert isinstance(obj, list)
-                index, howMany = argvals
-                res = []
-                for i in range(index, min(index + howMany, len(obj))):
-                    res.append(obj.pop(index))
-                return res
+                if obj == str:
+                    if member == 'fromCharCode':
+                        assertion(argvals, 'takes one or more arguments')
+                        return ''.join(map(chr, argvals))
+                    raise ExtractorError('Unsupported string method %s' % (member, ))
 
-            return obj[member](argvals)
+                if member == 'split':
+                    assertion(argvals, 'takes one or more arguments')
+                    assertion(argvals == [''], 'with arguments is not implemented')
+                    return list(obj)
+                elif member == 'join':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(len(argvals) == 1, 'takes exactly one argument')
+                    return argvals[0].join(obj)
+                elif member == 'reverse':
+                    assertion(not argvals, 'does not take any arguments')
+                    obj.reverse()
+                    return obj
+                elif member == 'slice':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(len(argvals) == 1, 'takes exactly one argument')
+                    return obj[argvals[0]:]
+                elif member == 'splice':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(argvals, 'takes one or more arguments')
+                    index, howMany = (argvals + [len(obj)])[:2]
+                    if index < 0:
+                        index += len(obj)
+                    add_items = argvals[2:]
+                    res = []
+                    for i in range(index, min(index + howMany, len(obj))):
+                        res.append(obj.pop(index))
+                    for i, item in enumerate(add_items):
+                        obj.insert(index + i, item)
+                    return res
+                elif member == 'unshift':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(argvals, 'takes one or more arguments')
+                    for item in reversed(argvals):
+                        obj.insert(0, item)
+                    return obj
+                elif member == 'pop':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    assertion(not argvals, 'does not take any arguments')
+                    if not obj:
+                        return
+                    return obj.pop()
+                elif member == 'push':
+                    assertion(argvals, 'takes one or more arguments')
+                    obj.extend(argvals)
+                    return obj
+                elif member == 'forEach':
+                    assertion(argvals, 'takes one or more arguments')
+                    assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
+                    f, this = (argvals + [''])[:2]
+                    return [f((item, idx, obj), this=this) for idx, item in enumerate(obj)]
+                elif member == 'indexOf':
+                    assertion(argvals, 'takes one or more arguments')
+                    assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
+                    idx, start = (argvals + [0])[:2]
+                    try:
+                        return obj.index(idx, start)
+                    except ValueError:
+                        return -1
 
-        for op, opfunc in _OPERATORS:
-            m = re.match(r'(?P<x>.+?)%s(?P<y>.+)' % re.escape(op), expr)
-            if not m:
-                continue
-            x, abort = self.interpret_statement(
-                m.group('x'), local_vars, allow_recursion - 1)
-            if abort:
-                raise ExtractorError(
-                    'Premature left-side return of %s in %r' % (op, expr))
-            y, abort = self.interpret_statement(
-                m.group('y'), local_vars, allow_recursion - 1)
-            if abort:
-                raise ExtractorError(
-                    'Premature right-side return of %s in %r' % (op, expr))
-            return opfunc(x, y)
+                if isinstance(obj, list):
+                    member = int(member)
+                    nl.member = member
+                return obj[member](argvals)
 
-        m = re.match(
-            r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]*)\)$' % _NAME_RE, expr)
+            if remaining:
+                return self.interpret_expression(
+                    self._named_object(local_vars, eval_method()) + remaining,
+                    local_vars, allow_recursion)
+            else:
+                return eval_method()
+
+        m = re.match(r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]*)\)$' % _NAME_RE, expr)
         if m:
             fname = m.group('func')
             argvals = tuple([
                 int(v) if v.isdigit() else local_vars[v]
-                for v in m.group('args').split(',')]) if len(m.group('args')) > 0 else tuple()
-            if fname not in self._functions:
+                for v in self._separate(m.group('args'))])
+            if fname in local_vars:
+                return local_vars[fname](argvals)
+            elif fname not in self._functions:
                 self._functions[fname] = self.extract_function(fname)
             return self._functions[fname](argvals)
 
-        raise ExtractorError('Unsupported JS expression %r' % expr)
+        if expr:
+            raise ExtractorError('Unsupported JS expression %r' % expr)
 
     def extract_object(self, objname):
         _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
@@ -233,30 +501,52 @@ class JSInterpreter(object):
 
         return obj
 
-    def extract_function(self, funcname):
+    def extract_function_code(self, funcname):
+        """ @returns argnames, code """
         func_m = re.search(
             r'''(?x)
-                (?:function\s+%s|[{;,]\s*%s\s*=\s*function|var\s+%s\s*=\s*function)\s*
+                (?:function\s+%(f_n)s|[{;,]\s*%(f_n)s\s*=\s*function|var\s+%(f_n)s\s*=\s*function)\s*
                 \((?P<args>[^)]*)\)\s*
-                \{(?P<code>[^}]+)\}''' % (
-                re.escape(funcname), re.escape(funcname), re.escape(funcname)),
+                (?P<code>\{(?:(?!};)[^"]|"([^"]|\\")*")+\})''' % {'f_n': re.escape(funcname), },
             self.code)
+        code, _ = self._separate_at_paren(func_m.group('code'), '}')  # refine the match
         if func_m is None:
             raise ExtractorError('Could not find JS function %r' % funcname)
-        argnames = func_m.group('args').split(',')
+        return func_m.group('args').split(','), code
 
-        return self.build_function(argnames, func_m.group('code'))
+    def extract_function(self, funcname):
+        return self.extract_function_from_code(*self.extract_function_code(funcname))
+
+    def extract_function_from_code(self, argnames, code, *global_stack):
+        local_vars = {}
+        while True:
+            mobj = re.search(r'function\((?P<args>[^)]*)\)\s*{', code)
+            if mobj is None:
+                break
+            start, body_start = mobj.span()
+            body, remaining = self._separate_at_paren(code[body_start - 1:], '}')
+            name = self._named_object(
+                local_vars,
+                self.extract_function_from_code(
+                    [str.strip(x) for x in mobj.group('args').split(',')],
+                    body, local_vars, *global_stack))
+            code = code[:start] + name + remaining
+        return self.build_function(argnames, code, local_vars, *global_stack)
 
     def call_function(self, funcname, *args):
-        f = self.extract_function(funcname)
-        return f(args)
+        return self.extract_function(funcname)(args)
 
-    def build_function(self, argnames, code):
-        def resf(args):
-            local_vars = dict(zip(argnames, args))
-            for stmt in code.split(';'):
-                res, abort = self.interpret_statement(stmt, local_vars)
-                if abort:
+    def build_function(self, argnames, code, *global_stack):
+        global_stack = list(global_stack) or [{}]
+        local_vars = global_stack.pop(0)
+
+        def resf(args, **kwargs):
+            local_vars.update(dict(zip(argnames, args)))
+            local_vars.update(kwargs)
+            var_stack = LocalNameSpace(local_vars, *global_stack)
+            for stmt in self._separate(code.replace('\n', ''), ';'):
+                ret, should_abort = self.interpret_statement(stmt, var_stack)
+                if should_abort:
                     break
-            return res
+            return ret
         return resf

From e1eae16b56b5c57e341b000167c0a92e67095e6e Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Thu, 4 Nov 2021 12:48:06 +0000
Subject: [PATCH 433/860] Handle default in switch better

Add https://github.com/yt-dlp/yt-dlp/commit/a1fc7ca0743c8df06416e68ee74b64e07dfe7135
Thanks coletdjnz
---
 test/test_jsinterp.py  | 15 +++++++++++++++
 youtube_dl/jsinterp.py | 23 ++++++++++++++---------
 2 files changed, 29 insertions(+), 9 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 4d05ea610..acdabffb1 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -133,6 +133,21 @@ class TestJSInterpreter(unittest.TestCase):
         self.assertEqual(jsi.call_function('x', 3), 6)
         self.assertEqual(jsi.call_function('x', 5), 0)
 
+    def test_switch_default(self):
+        jsi = JSInterpreter('''
+        function x(f) { switch(f){
+            case 2: f+=2;
+            default: f-=1;
+            case 5:
+            case 6: f+=6;
+            case 0: break;
+            case 1: f+=1;
+        } return f }
+        ''')
+        self.assertEqual(jsi.call_function('x', 1), 2)
+        self.assertEqual(jsi.call_function('x', 5), 11)
+        self.assertEqual(jsi.call_function('x', 9), 14)
+
     def test_try(self):
         jsi = JSInterpreter('''
         function x() { try{return 10} catch(e){return 5} }
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 061e92c2a..c35765702 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -240,21 +240,26 @@ class JSInterpreter(object):
             switch_val, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
             switch_val = self.interpret_expression(switch_val, local_vars, allow_recursion)
             body, expr = self._separate_at_paren(remaining, '}')
-            body, default = body.split('default:') if 'default:' in body else (body, None)
-            items = body.split('case ')[1:]
-            if default:
-                items.append('default:%s' % (default, ))
-            matched = False
-            for item in items:
-                case, stmt = [i.strip() for i in self._separate(item, ':', 1)]
-                matched = matched or case == 'default' or switch_val == self.interpret_expression(case, local_vars, allow_recursion)
-                if matched:
+            items = body.replace('default:', 'case default:').split('case ')[1:]
+            for default in (False, True):
+                matched = False
+                for item in items:
+                    case, stmt = [i.strip() for i in self._separate(item, ':', 1)]
+                    if default:
+                        matched = matched or case == 'default'
+                    elif not matched:
+                        matched = (case != 'default'
+                                   and switch_val == self.interpret_expression(case, local_vars, allow_recursion))
+                    if not matched:
+                        continue
                     try:
                         ret, should_abort = self.interpret_statement(stmt, local_vars, allow_recursion - 1)
                         if should_abort:
                             return ret
                     except JS_Break:
                         break
+                if matched:
+                    break
             return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
 
         # Comma separated statements

From 1ca673bd98cc5bbfa76d00ac84ad5f6c1376db01 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 27 Nov 2021 02:06:13 +0000
Subject: [PATCH 434/860] Fix splice to handle float

Needed for new youtube js player f1ca6900
Add https://github.com/yt-dlp/yt-dlp/commit/57dbe8077f8d00e0fffac53669f40cd7d584474f#diff-729b57caa8d006426f6a8960c061f519a8b6658682284015e069745af52ffb07
---
 youtube_dl/jsinterp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index c35765702..c75cf45b9 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -416,7 +416,7 @@ class JSInterpreter(object):
                 elif member == 'splice':
                     assertion(isinstance(obj, list), 'must be applied on a list')
                     assertion(argvals, 'takes one or more arguments')
-                    index, howMany = (argvals + [len(obj)])[:2]
+                    index, howMany = map(int, (argvals + [len(obj)])[:2])
                     if index < 0:
                         index += len(obj)
                     add_items = argvals[2:]

From 9d142109f445ea247e476cfc0e0ca134f6ebb802 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 27 Nov 2021 03:18:29 +0000
Subject: [PATCH 435/860] Back-port test_youtube_signature.py from yt-dlp and
 fix JSInterp accordingly

---
 test/test_youtube_signature.py | 89 ++++++++++++++++++++++++----------
 youtube_dl/jsinterp.py         |  9 ++--
 2 files changed, 69 insertions(+), 29 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 627d4cb92..c8e85b500 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -14,9 +14,10 @@ import string
 
 from test.helper import FakeYDL
 from youtube_dl.extractor import YoutubeIE
+from youtube_dl.jsinterp import JSInterpreter
 from youtube_dl.compat import compat_str, compat_urlretrieve
 
-_TESTS = [
+_SIG_TESTS = [
     (
         'https://s.ytimg.com/yts/jsbin/html5player-vflHOr_nV.js',
         86,
@@ -64,6 +65,25 @@ _TESTS = [
     )
 ]
 
+_NSIG_TESTS = [
+    (
+        'https://www.youtube.com/s/player/9216d1f7/player_ias.vflset/en_US/base.js',
+        'SLp9F5bwjAdhE9F-', 'gWnb9IK2DJ8Q1w',
+    ),
+    (
+        'https://www.youtube.com/s/player/f8cb7a3b/player_ias.vflset/en_US/base.js',
+        'oBo2h5euWy6osrUt', 'ivXHpm7qJjJN',
+    ),
+    (
+        'https://www.youtube.com/s/player/2dfe380c/player_ias.vflset/en_US/base.js',
+        'oBo2h5euWy6osrUt', '3DIBbn3qdQ',
+    ),
+    (
+        'https://www.youtube.com/s/player/f1ca6900/player_ias.vflset/en_US/base.js',
+        'cu3wyu6LQn2hse', 'jvxetvmlI9AN9Q',
+    ),
+]
+
 
 class TestPlayerInfo(unittest.TestCase):
     def test_youtube_extract_player_info(self):
@@ -95,35 +115,54 @@ class TestSignature(unittest.TestCase):
             os.mkdir(self.TESTDATA_DIR)
 
 
-def make_tfunc(url, sig_input, expected_sig):
-    m = re.match(r'.*-([a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$', url)
-    assert m, '%r should follow URL format' % url
-    test_id = m.group(1)
+def t_factory(name, sig_func, url_pattern):
+    def make_tfunc(url, sig_input, expected_sig):
+        m = url_pattern.match(url)
+        assert m, '%r should follow URL format' % url
+        test_id = m.group('id')
 
-    def test_func(self):
-        basename = 'player-%s.js' % test_id
-        fn = os.path.join(self.TESTDATA_DIR, basename)
+        def test_func(self):
+            basename = 'player-{0}-{1}.js'.format(name, test_id)
+            fn = os.path.join(self.TESTDATA_DIR, basename)
 
-        if not os.path.exists(fn):
-            compat_urlretrieve(url, fn)
+            if not os.path.exists(fn):
+                compat_urlretrieve(url, fn)
+            with io.open(fn, encoding='utf-8') as testf:
+                jscode = testf.read()
+            self.assertEqual(sig_func(jscode, sig_input), expected_sig)
 
-        ydl = FakeYDL()
-        ie = YoutubeIE(ydl)
-        with io.open(fn, encoding='utf-8') as testf:
-            jscode = testf.read()
-        func = ie._parse_sig_js(jscode)
-        src_sig = (
-            compat_str(string.printable[:sig_input])
-            if isinstance(sig_input, int) else sig_input)
-        got_sig = func(src_sig)
-        self.assertEqual(got_sig, expected_sig)
-
-    test_func.__name__ = str('test_signature_js_' + test_id)
-    setattr(TestSignature, test_func.__name__, test_func)
+        test_func.__name__ = str('test_{0}_js_{1}'.format(name, test_id))
+        setattr(TestSignature, test_func.__name__, test_func)
+    return make_tfunc
 
 
-for test_spec in _TESTS:
-    make_tfunc(*test_spec)
+def signature(jscode, sig_input):
+    func = YoutubeIE(FakeYDL())._parse_sig_js(jscode)
+    src_sig = (
+        compat_str(string.printable[:sig_input])
+        if isinstance(sig_input, int) else sig_input)
+    return func(src_sig)
+
+
+def n_sig(jscode, sig_input):
+    # Pending implementation of _extract_n_function_name() or similar in
+    # youtube.py, hard-code here
+    # funcname = YoutubeIE(FakeYDL())._extract_n_function_name(jscode)
+    import re
+    funcname = re.search(r'[=(,&|](\w+)\(\w+\),\w+\.set\("n",', jscode)
+    funcname = funcname and funcname.group(1)
+    return JSInterpreter(jscode).call_function(funcname, sig_input)
+
+
+make_sig_test = t_factory(
+    'signature', signature, re.compile(r'.*-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$'))
+for test_spec in _SIG_TESTS:
+    make_sig_test(*test_spec)
+
+make_nsig_test = t_factory(
+    'nsig', n_sig, re.compile(r'.+/player/(?P<id>[a-zA-Z0-9_-]+)/.+.js$'))
+for test_spec in _NSIG_TESTS:
+    make_nsig_test(*test_spec)
 
 
 if __name__ == '__main__':
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index c75cf45b9..a2306557b 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -9,7 +9,8 @@ from .utils import (
     remove_quotes,
 )
 from .compat import (
-    compat_collections_abc
+    compat_collections_abc,
+    compat_str,
 )
 MutableMapping = compat_collections_abc.MutableMapping
 
@@ -372,7 +373,7 @@ class JSInterpreter(object):
                 # nonlocal member
                 member = nl.member
                 if variable == 'String':
-                    obj = str
+                    obj = compat_str
                 elif variable in local_vars:
                     obj = local_vars[variable]
                 else:
@@ -391,7 +392,7 @@ class JSInterpreter(object):
                     self.interpret_expression(v, local_vars, allow_recursion)
                     for v in self._separate(arg_str)]
 
-                if obj == str:
+                if obj == compat_str:
                     if member == 'fromCharCode':
                         assertion(argvals, 'takes one or more arguments')
                         return ''.join(map(chr, argvals))
@@ -533,7 +534,7 @@ class JSInterpreter(object):
             name = self._named_object(
                 local_vars,
                 self.extract_function_from_code(
-                    [str.strip(x) for x in mobj.group('args').split(',')],
+                    [x.strip() for x in mobj.group('args').split(',')],
                     body, local_vars, *global_stack))
             code = code[:start] + name + remaining
         return self.build_function(argnames, code, local_vars, *global_stack)

From 6ca7b776965ed1e9220690edc4ee22de8c8587f5 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 10 Dec 2021 19:14:54 +0000
Subject: [PATCH 436/860] Refactor JSInterpreter._separate

yt-dlp/yt-dlp/@06dfe0a, improve _MATCHING_PARENS
---
 youtube_dl/jsinterp.py | 36 ++++++++++++++++++------------------
 1 file changed, 18 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index a2306557b..8eaa911cd 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -36,6 +36,8 @@ _ASSIGN_OPERATORS.append(('=', (lambda cur, right: right)))
 
 _NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*'
 
+_MATCHING_PARENS = dict(zip(*zip('()', '{}', '[]')))
+
 
 class JS_Break(ExtractorError):
     def __init__(self):
@@ -100,26 +102,24 @@ class JSInterpreter(object):
     def _separate(expr, delim=',', max_split=None):
         if not expr:
             return
-        parens = {'(': 0, '{': 0, '[': 0, ']': 0, '}': 0, ')': 0}
-        start, splits, pos, max_pos = 0, 0, 0, len(delim) - 1
+        counters = {k: 0 for k in _MATCHING_PARENS.values()}
+        start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
         for idx, char in enumerate(expr):
-            if char in parens:
-                parens[char] += 1
-            is_in_parens = (parens['['] - parens[']']
-                            or parens['('] - parens[')']
-                            or parens['{'] - parens['}'])
-            if char == delim[pos] and not is_in_parens:
-                if pos == max_pos:
-                    pos = 0
-                    yield expr[start: idx - max_pos]
-                    start = idx + 1
-                    splits += 1
-                    if max_split and splits >= max_split:
-                        break
-                else:
-                    pos += 1
-            else:
+            if char in _MATCHING_PARENS:
+                counters[_MATCHING_PARENS[char]] += 1
+            elif char in counters:
+                counters[char] -= 1
+            if char != delim[pos] or any(counters.values()):
                 pos = 0
+                continue
+            elif pos != delim_len:
+                pos += 1
+                continue
+            yield expr[start: idx - delim_len]
+            start, pos = idx + 1, 0
+            splits += 1
+            if max_split and splits >= max_split:
+                break
         yield expr[start:]
 
     @staticmethod

From af9e72507ea38e5ab3fa2751ed09ec88021260cb Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Mon, 1 Nov 2021 04:45:42 +0000
Subject: [PATCH 437/860] Implement n-param descrambling using JSInterp

Fixes #29326, closes #29790, closes #30004, closes #30024, closes #30052,
closes #30088, closes #30097, closes #30102, closes #30109, closes #30119,
closes #30125, closes #30128, closes #30162, closes #30173, closes #30186,
closes #30192, closes #30221, closes #30239, closes #30539, closes #30552.
---
 youtube_dl/extractor/youtube.py | 115 +++++++++++++++++++++++++++-----
 1 file changed, 99 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index da410f8f0..63918924d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1254,6 +1254,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             raise ExtractorError('Cannot identify player %r' % player_url)
         return id_m.group('id')
 
+    def _get_player_code(self, video_id, player_url, player_id=None):
+        if not player_id:
+            player_id = self._extract_player_info(player_url)
+
+        if player_id not in self._code_cache:
+            self._code_cache[player_id] = self._download_webpage(
+                player_url, video_id,
+                note='Downloading player ' + player_id,
+                errnote='Download of %s failed' % player_url)
+        return self._code_cache[player_id]
+
     def _extract_signature_function(self, video_id, player_url, example_sig):
         player_id = self._extract_player_info(player_url)
 
@@ -1266,12 +1277,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if cache_spec is not None:
             return lambda s: ''.join(s[i] for i in cache_spec)
 
-        if player_id not in self._code_cache:
-            self._code_cache[player_id] = self._download_webpage(
-                player_url, video_id,
-                note='Downloading player ' + player_id,
-                errnote='Download of %s failed' % player_url)
-        code = self._code_cache[player_id]
+        code = self._get_player_code(video_id, player_url, player_id)
         res = self._parse_sig_js(code)
 
         test_string = ''.join(map(compat_chr, range(len(example_sig))))
@@ -1350,11 +1356,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if player_url is None:
             raise ExtractorError('Cannot decrypt signature without player_url')
 
-        if player_url.startswith('//'):
-            player_url = 'https:' + player_url
-        elif not re.match(r'https?://', player_url):
-            player_url = compat_urlparse.urljoin(
-                'https://www.youtube.com', player_url)
         try:
             player_id = (player_url, self._signature_cache_id(s))
             if player_id not in self._player_cache:
@@ -1371,6 +1372,88 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             raise ExtractorError(
                 'Signature extraction failed: ' + tb, cause=e)
 
+    def _extract_player_url(self, webpage):
+        player_url = self._search_regex(
+            r'"(?:PLAYER_JS_URL|jsUrl)"\s*:\s*"([^"]+)"',
+            webpage or '', 'player URL', fatal=False)
+        if not player_url:
+            return
+        if player_url.startswith('//'):
+            player_url = 'https:' + player_url
+        elif not re.match(r'https?://', player_url):
+            player_url = compat_urlparse.urljoin(
+                'https://www.youtube.com', player_url)
+        return player_url
+
+    # from yt-dlp
+    # See also:
+    # 1. https://github.com/ytdl-org/youtube-dl/issues/29326#issuecomment-894619419
+    # 2. https://code.videolan.org/videolan/vlc/-/blob/4fb284e5af69aa9ac2100ccbdd3b88debec9987f/share/lua/playlist/youtube.lua#L116
+    # 3. https://github.com/ytdl-org/youtube-dl/issues/30097#issuecomment-950157377
+    def _extract_n_function_name(self, jscode):
+        return self._search_regex(
+            (r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]{3})\([a-zA-Z0-9]\)',),
+            jscode, 'Initial JS player n function name', group='nfunc')
+
+    def _extract_n_function(self, video_id, player_url):
+        player_id = self._extract_player_info(player_url)
+        func_code = self._downloader.cache.load('youtube-nsig', player_id)
+
+        if func_code:
+            jsi = JSInterpreter(func_code)
+        else:
+            player_id = self._extract_player_info(player_url)
+            jscode = self._get_player_code(video_id, player_url, player_id)
+            funcname = self._extract_n_function_name(jscode)
+            jsi = JSInterpreter(jscode)
+            func_code = jsi.extract_function_code(funcname)
+            self._downloader.cache.store('youtube-nsig', player_id, func_code)
+
+        if self._downloader.params.get('youtube_print_sig_code'):
+            self.to_screen('Extracted nsig function from {0}:\n{1}\n'.format(player_id, func_code[1]))
+
+        return lambda s: jsi.extract_function_from_code(*func_code)([s])
+
+    def _n_descramble(self, n_param, player_url, video_id):
+        """Compute the response to YT's "n" parameter challenge
+
+        Args:
+        n_param     -- challenge string that is the value of the
+                       URL's "n" query parameter
+        player_url  -- URL of YT player JS
+        video_id
+        """
+
+        sig_id = ('nsig_value', n_param)
+        if sig_id in self._player_cache:
+            return self._player_cache[sig_id]
+
+        try:
+            player_id = ('nsig', player_url)
+            if player_id not in self._player_cache:
+                self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
+            func = self._player_cache[player_id]
+            self._player_cache[sig_id] = func(n_param)
+            if self._downloader.params.get('verbose', False):
+                self._downloader.to_screen('[debug] [%s] %s' % (self.IE_NAME, 'Decrypted nsig {0} => {1}'.format(n_param, self._player_cache[sig_id])))
+            return self._player_cache[sig_id]
+        except Exception as e:
+            raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
+
+    def _unthrottle_format_urls(self, video_id, player_url, formats):
+        for fmt in formats:
+            parsed_fmt_url = compat_urlparse.urlparse(fmt['url'])
+            qs = compat_urlparse.parse_qs(parsed_fmt_url.query)
+            n_param = qs.get('n')
+            if not n_param:
+                continue
+            n_param = n_param[-1]
+            n_response = self._n_descramble(n_param, player_url, video_id)
+            if n_response:
+                qs['n'] = [n_response]
+                fmt['url'] = compat_urlparse.urlunparse(
+                    parsed_fmt_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
+
     def _mark_watched(self, video_id, player_response):
         playback_url = url_or_none(try_get(
             player_response,
@@ -1632,11 +1715,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 if not (sc and fmt_url and encrypted_sig):
                     continue
                 if not player_url:
-                    if not webpage:
-                        continue
-                    player_url = self._search_regex(
-                        r'"(?:PLAYER_JS_URL|jsUrl)"\s*:\s*"([^"]+)"',
-                        webpage, 'player URL', fatal=False)
+                    player_url = self._extract_player_url(webpage)
                 if not player_url:
                     continue
                 signature = self._decrypt_signature(sc['s'][0], video_id, player_url)
@@ -1782,6 +1861,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         is_live = video_details.get('isLive')
         owner_profile_url = microformat.get('ownerProfileUrl')
 
+        if not player_url:
+            player_url = self._extract_player_url(webpage)
+        self._unthrottle_format_urls(video_id, player_url, formats)
+
         info = {
             'id': video_id,
             'title': self._live_title(video_title) if is_live else video_title,

From 1e677567cd083d43f55daef0cc74e5fa24575ae3 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 1 Feb 2022 14:39:03 +0000
Subject: [PATCH 438/860] [YouTube] Fix n-sig for player e06dea74 (#30582)

From yt-dl commit 48416bc
---
 test/test_youtube_signature.py  | 24 +++++++++++++++++-------
 youtube_dl/extractor/youtube.py | 14 +++++++++++---
 2 files changed, 28 insertions(+), 10 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index c8e85b500..fc5e9828e 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -82,6 +82,14 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/f1ca6900/player_ias.vflset/en_US/base.js',
         'cu3wyu6LQn2hse', 'jvxetvmlI9AN9Q',
     ),
+    (
+        'https://www.youtube.com/s/player/8040e515/player_ias.vflset/en_US/base.js',
+        'wvOFaY-yjgDuIEg5', 'HkfBFDHmgw4rsw',
+    ),
+    (
+        'https://www.youtube.com/s/player/e06dea74/player_ias.vflset/en_US/base.js',
+        'AiuodmaDDYw8d3y4bf', 'ankd8eza2T6Qmw',
+    ),
 ]
 
 
@@ -110,10 +118,17 @@ class TestPlayerInfo(unittest.TestCase):
 class TestSignature(unittest.TestCase):
     def setUp(self):
         TEST_DIR = os.path.dirname(os.path.abspath(__file__))
-        self.TESTDATA_DIR = os.path.join(TEST_DIR, 'testdata')
+        self.TESTDATA_DIR = os.path.join(TEST_DIR, 'testdata/sigs')
         if not os.path.exists(self.TESTDATA_DIR):
             os.mkdir(self.TESTDATA_DIR)
 
+    def tearDown(self):
+        try:
+            for f in os.listdir(self.TESTDATA_DIR):
+                os.remove(f)
+        except OSError:
+            pass
+
 
 def t_factory(name, sig_func, url_pattern):
     def make_tfunc(url, sig_input, expected_sig):
@@ -145,12 +160,7 @@ def signature(jscode, sig_input):
 
 
 def n_sig(jscode, sig_input):
-    # Pending implementation of _extract_n_function_name() or similar in
-    # youtube.py, hard-code here
-    # funcname = YoutubeIE(FakeYDL())._extract_n_function_name(jscode)
-    import re
-    funcname = re.search(r'[=(,&|](\w+)\(\w+\),\w+\.set\("n",', jscode)
-    funcname = funcname and funcname.group(1)
+    funcname = YoutubeIE(FakeYDL())._extract_n_function_name(jscode)
     return JSInterpreter(jscode).call_function(funcname, sig_input)
 
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 63918924d..7943b94f9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -28,6 +28,7 @@ from ..utils import (
     dict_get,
     float_or_none,
     int_or_none,
+    js_to_json,
     mimetype2ext,
     parse_codecs,
     parse_duration,
@@ -1391,9 +1392,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
     # 2. https://code.videolan.org/videolan/vlc/-/blob/4fb284e5af69aa9ac2100ccbdd3b88debec9987f/share/lua/playlist/youtube.lua#L116
     # 3. https://github.com/ytdl-org/youtube-dl/issues/30097#issuecomment-950157377
     def _extract_n_function_name(self, jscode):
-        return self._search_regex(
-            (r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z0-9$]{3})\([a-zA-Z0-9]\)',),
-            jscode, 'Initial JS player n function name', group='nfunc')
+        target = r'(?P<nfunc>[a-zA-Z0-9$]{3})(?:\[(?P<idx>\d+)\])?'
+        nfunc_and_idx = self._search_regex(
+            r'\.get\("n"\)\)&&\(b=(%s)\([a-zA-Z0-9]\)' % (target, ),
+            jscode, 'Initial JS player n function name')
+        nfunc, idx = re.match(target, nfunc_and_idx).group('nfunc', 'idx')
+        if not idx:
+            return nfunc
+        return self._parse_json(self._search_regex(
+            r'var %s\s*=\s*(\[.+?\]);' % (nfunc, ), jscode,
+            'Initial JS player n function list ({nfunc}[{idx}])'.format(**locals())), nfunc, transform_source=js_to_json)[int(idx)]
 
     def _extract_n_function(self, video_id, player_url):
         player_id = self._extract_player_info(player_url)

From 34c06b16f5eb814308392b68dce07bbff62bc406 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 31 Jan 2022 00:02:56 +0000
Subject: [PATCH 439/860] Support Youtube Shorts URL format

---
 youtube_dl/extractor/youtube.py | 17 +++++++++++++++++
 1 file changed, 17 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 7943b94f9..05688dc70 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -417,6 +417,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                          (?:.*?\#/)?                                          # handle anchor (#/) redirect urls
                          (?:                                                  # the various things that can precede the ID:
                              (?:(?:v|embed|e)/(?!videoseries))                # v/ or embed/ or e/
+                             |shorts/
                              |(?:                                             # or the v= param in all its forms
                                  (?:(?:watch|movie)(?:_popup)?(?:\.php)?/?)?  # preceding watch(_popup|.php) or nothing (like /?v=xxxx)
                                  (?:\?|\#!?)                                  # the params delimiter ? or # or #!
@@ -1119,6 +1120,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': True,
             },
         },
+        {
+            # YT 'Shorts'
+            'url': 'https://youtube.com/shorts/4L2J27mJ3Dc',
+            'info_dict': {
+                'id': '4L2J27mJ3Dc',
+                'ext': 'mp4',
+                'upload_date': '20211025',
+                'uploader': 'Charlie Berens',
+                'description': 'md5:976512b8a29269b93bbd8a61edc45a6d',
+                'uploader_id': 'fivedlrmilkshake',
+                'title': 'Midwest Squid Game #Shorts',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        },
     ]
     _formats = {
         '5': {'ext': 'flv', 'width': 400, 'height': 240, 'acodec': 'mp3', 'abr': 64, 'vcodec': 'h263'},

From 41f0043983c831b7c0c3614340d2f66ec153087b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 1 Feb 2022 23:22:57 +0000
Subject: [PATCH 440/860] Avoid crashing if n-sig decode fails

---
 youtube_dl/extractor/youtube.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 05688dc70..4165de15c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -26,6 +26,7 @@ from ..utils import (
     ExtractorError,
     clean_html,
     dict_get,
+    error_to_compat_str,
     float_or_none,
     int_or_none,
     js_to_json,
@@ -1463,7 +1464,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 self._downloader.to_screen('[debug] [%s] %s' % (self.IE_NAME, 'Decrypted nsig {0} => {1}'.format(n_param, self._player_cache[sig_id])))
             return self._player_cache[sig_id]
         except Exception as e:
-            raise ExtractorError(traceback.format_exc(), cause=e, video_id=video_id)
+            self._downloader.report_warning(
+                '[%s] %s (%s %s)' % (
+                    self.IE_NAME,
+                    'Unable to decode n-parameter: download likely to be throttled',
+                    error_to_compat_str(e),
+                    traceback.format_exc()))
 
     def _unthrottle_format_urls(self, video_id, player_url, formats):
         for fmt in formats:

From 78ce962f4fe020994c216dd2671546fbe58a5c67 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 30 Jan 2022 01:24:09 +0530
Subject: [PATCH 441/860] [youtube] Support channel search

Code from https://github.com/yt-dlp/yt-dlp/commit/cd684175adbe663bbdf6a6c72d8b99b617b6ff2e
---
 youtube_dl/extractor/youtube.py | 18 ++++++++++++++++--
 1 file changed, 16 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4165de15c..8e1254f19 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2438,6 +2438,17 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
     }, {
         'url': 'https://www.youtube.com/watch?list=PLW4dVinRY435CBE_JD3t-0SRXKfnZHS1P&feature=youtu.be&v=M9cJMXmQ_ZU',
         'only_matching': True,
+    }, {
+        'note': 'Search tab',
+        'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
+        'playlist_mincount': 40,
+        'info_dict': {
+            'id': 'UCYO_jab_esuFRV4b17AJtAw',
+            'title': '3Blue1Brown - Search - linear algebra',
+            'description': 'md5:e1384e8a133307dd10edee76e875d62f',
+            'uploader': '3Blue1Brown',
+            'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
+        }
     }]
 
     @classmethod
@@ -2835,8 +2846,9 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
     @staticmethod
     def _extract_selected_tab(tabs):
         for tab in tabs:
-            if try_get(tab, lambda x: x['tabRenderer']['selected'], bool):
-                return tab['tabRenderer']
+            renderer = dict_get(tab, ('tabRenderer', 'expandableTabRenderer')) or {}
+            if renderer.get('selected') is True:
+                return renderer
         else:
             raise ExtractorError('Unable to find selected tab')
 
@@ -2893,6 +2905,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             title = channel_title or item_id
             if tab_title:
                 title += ' - %s' % tab_title
+            if selected_tab.get('expandedText'):
+                title += ' - %s' % selected_tab['expandedText']
             description = renderer.get('description')
             playlist_id = renderer.get('externalId')
         else:

From 7a497f1405ecdcd76c671c7bfaad238d75d01639 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Feb 2022 04:09:23 +0000
Subject: [PATCH 442/860] Rework 2c2c2bd with an actual Mix page and realistic
 playlist size

From https://github.com/ytdl-org/youtube-dl/commit/2c2c2bd348b7dce0aad55a6fc37a18c6f9a000e3#commitcomment-65953545
---
 test/test_youtube_lists.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/test/test_youtube_lists.py b/test/test_youtube_lists.py
index 07a6b6d06..e0e8891ba 100644
--- a/test/test_youtube_lists.py
+++ b/test/test_youtube_lists.py
@@ -36,12 +36,12 @@ class TestYoutubeLists(unittest.TestCase):
         dl = FakeYDL()
         dl.params['format'] = 'best'
         ie = YoutubeTabIE(dl)
-        result = dl.extract_info('https://www.youtube.com/watch?v=uVJ0Il5WvbE&list=PLhQjrBD2T381k8ul4WQ8SQ165XqY149WW',
+        result = dl.extract_info('https://www.youtube.com/watch?v=tyITL_exICo&list=RDCLAK5uy_kLWIr9gv1XLlPbaDS965-Db4TrBoUTxQ8',
                                  download=False, ie_key=ie.ie_key(), process=True)
         entries = (result or {}).get('entries', [{'id': 'not_found', }])
-        self.assertTrue(len(entries) >= 50)
+        self.assertTrue(len(entries) >= 25)
         original_video = entries[0]
-        self.assertEqual(original_video['id'], 'uVJ0Il5WvbE')
+        self.assertEqual(original_video['id'], 'tyITL_exICo')
 
     def test_youtube_flat_playlist_extraction(self):
         dl = FakeYDL()

From 0c0876f790c78c38ececbc920073e8b6cf01e9c7 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 3 Feb 2022 07:44:37 +0530
Subject: [PATCH 443/860] [youtube:search] Add tests

---
 youtube_dl/extractor/youtube.py | 21 ++++++++++++++++++---
 1 file changed, 18 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3ab60960a..41695a561 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -3206,7 +3206,14 @@ class YoutubeSearchIE(SearchInfoExtractor, YoutubeBaseInfoExtractor):
     _SEARCH_KEY = 'ytsearch'
     _SEARCH_PARAMS = 'EgIQAQ%3D%3D'  # Videos only
     _MAX_RESULTS = float('inf')
-    _TESTS = []
+    _TESTS = [{
+        'url': 'ytsearch10:youtube-dl test video',
+        'playlist_count': 10,
+        'info_dict': {
+            'id': 'youtube-dl test video',
+            'title': 'youtube-dl test video',
+        }
+    }]
 
     def _get_n_results(self, query, n):
         """Get a specified number of results for a query"""
@@ -3219,7 +3226,14 @@ class YoutubeSearchDateIE(YoutubeSearchIE):
     _SEARCH_KEY = 'ytsearchdate'
     IE_DESC = 'YouTube.com searches, newest videos first'
     _SEARCH_PARAMS = 'CAISAhAB'  # Videos only, sorted by date
-    _TESTS = []
+    _TESTS = [{
+        'url': 'ytsearchdate10:youtube-dl test video',
+        'playlist_count': 10,
+        'info_dict': {
+            'id': 'youtube-dl test video',
+            'title': 'youtube-dl test video',
+        }
+    }]
 
 
 class YoutubeSearchURLIE(YoutubeBaseInfoExtractor):
@@ -3232,7 +3246,8 @@ class YoutubeSearchURLIE(YoutubeBaseInfoExtractor):
         'info_dict': {
             'id': 'youtube-dl test video',
             'title': 'youtube-dl test video',
-        }
+        },
+        'params': {'playlistend': 5}
     }, {
         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
         'only_matching': True,

From 61d791726f67255c2ed3c0bb6ee24c8c1faeb028 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Feb 2022 11:24:03 +0000
Subject: [PATCH 444/860] Find TV2DK Kaltura ID in Nuxt.js page format

---
 youtube_dl/extractor/tv2dk.py | 15 ++++++++++++---
 1 file changed, 12 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/tv2dk.py b/youtube_dl/extractor/tv2dk.py
index 8bd5fd640..106a081e1 100644
--- a/youtube_dl/extractor/tv2dk.py
+++ b/youtube_dl/extractor/tv2dk.py
@@ -41,8 +41,16 @@ class TV2DKIE(InfoExtractor):
             'duration': 1347,
             'view_count': int,
         },
-        'params': {
-            'skip_download': True,
+        'add_ie': ['Kaltura'],
+    }, {
+        'url': 'https://www.tv2lorry.dk/gadekamp/gadekamp-6-hoejhuse-i-koebenhavn',
+        'info_dict': {
+            'id': '1_7iwll9n0',
+            'ext': 'mp4',
+            'upload_date': '20211027',
+            'title': 'Gadekamp #6 - Højhuse i København',
+            'uploader_id': 'tv2lorry',
+            'timestamp': 1635345229,
         },
         'add_ie': ['Kaltura'],
     }, {
@@ -91,7 +99,8 @@ class TV2DKIE(InfoExtractor):
             add_entry(partner_id, kaltura_id)
         if not entries:
             kaltura_id = self._search_regex(
-                r'entry_id\s*:\s*["\']([0-9a-z_]+)', webpage, 'kaltura id')
+                (r'entry_id\s*:\s*["\']([0-9a-z_]+)',
+                 r'\\u002FentryId\\u002F(\w+)\\u002F'), webpage, 'kaltura id')
             partner_id = self._search_regex(
                 (r'\\u002Fp\\u002F(\d+)\\u002F', r'/p/(\d+)/'), webpage,
                 'partner id')

From 27dbf6f0ab778a9e3d81be64a615046e6737c3f6 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Feb 2022 11:38:44 +0000
Subject: [PATCH 445/860] Return the item itself if playlist has one entry

Removes playlist spam from log
---
 youtube_dl/extractor/tv2dk.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/tv2dk.py b/youtube_dl/extractor/tv2dk.py
index 106a081e1..ec5cbdf03 100644
--- a/youtube_dl/extractor/tv2dk.py
+++ b/youtube_dl/extractor/tv2dk.py
@@ -105,6 +105,8 @@ class TV2DKIE(InfoExtractor):
                 (r'\\u002Fp\\u002F(\d+)\\u002F', r'/p/(\d+)/'), webpage,
                 'partner id')
             add_entry(partner_id, kaltura_id)
+        if len(entries) == 1:
+            return entries[0]
         return self.playlist_result(entries)
 
 

From 8248133e5ee5579316120cbcbff3ba8b713f1017 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Feb 2022 11:29:41 +0000
Subject: [PATCH 446/860] Back-port yt-dlp Viki extractor

From https://github.com/yt-dlp/yt-dlp/pull/2540
---
 youtube_dl/extractor/viki.py | 335 +++++++++++++++--------------------
 1 file changed, 144 insertions(+), 191 deletions(-)

diff --git a/youtube_dl/extractor/viki.py b/youtube_dl/extractor/viki.py
index 2e9cbf148..2ddca0ca6 100644
--- a/youtube_dl/extractor/viki.py
+++ b/youtube_dl/extractor/viki.py
@@ -1,38 +1,29 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import base64
 import hashlib
 import hmac
-import itertools
 import json
-import re
 import time
 
 from .common import InfoExtractor
-from ..compat import (
-    compat_parse_qs,
-    compat_urllib_parse_urlparse,
-)
 from ..utils import (
     ExtractorError,
     int_or_none,
     parse_age_limit,
     parse_iso8601,
-    sanitized_Request,
-    std_headers,
     try_get,
 )
 
 
 class VikiBaseIE(InfoExtractor):
     _VALID_URL_BASE = r'https?://(?:www\.)?viki\.(?:com|net|mx|jp|fr)/'
-    _API_QUERY_TEMPLATE = '/v4/%sapp=%s&t=%s&site=www.viki.com'
-    _API_URL_TEMPLATE = 'https://api.viki.io%s&sig=%s'
+    _API_URL_TEMPLATE = 'https://api.viki.io%s'
 
+    _DEVICE_ID = '112395910d'
     _APP = '100005a'
-    _APP_VERSION = '6.0.0'
-    _APP_SECRET = 'MM_d*yP@`&1@]@!AVrXf_o-HVEnoTnm$O-ti4[G~$JDI/Dc-&piU&z&5.;:}95=Iad'
+    _APP_VERSION = '6.11.3'
+    _APP_SECRET = 'd96704b180208dbb2efa30fe44c48bd8690441af9f567ba8fd710a72badc85198f7472'
 
     _GEO_BYPASS = False
     _NETRC_MACHINE = 'viki'
@@ -45,43 +36,60 @@ class VikiBaseIE(InfoExtractor):
         'paywall': 'Sorry, this content is only available to Viki Pass Plus subscribers',
     }
 
-    def _prepare_call(self, path, timestamp=None, post_data=None):
+    def _stream_headers(self, timestamp, sig):
+        return {
+            'X-Viki-manufacturer': 'vivo',
+            'X-Viki-device-model': 'vivo 1606',
+            'X-Viki-device-os-ver': '6.0.1',
+            'X-Viki-connection-type': 'WIFI',
+            'X-Viki-carrier': '',
+            'X-Viki-as-id': '100005a-1625321982-3932',
+            'timestamp': str(timestamp),
+            'signature': str(sig),
+            'x-viki-app-ver': self._APP_VERSION
+        }
+
+    def _api_query(self, path, version=4, **kwargs):
         path += '?' if '?' not in path else '&'
-        if not timestamp:
-            timestamp = int(time.time())
-        query = self._API_QUERY_TEMPLATE % (path, self._APP, timestamp)
+        app = self._APP
+        query = '/v{version}/{path}app={app}'.format(**locals())
         if self._token:
             query += '&token=%s' % self._token
+        return query + ''.join('&{name}={val}.format(**locals())' for name, val in kwargs.items())
+
+    def _sign_query(self, path):
+        timestamp = int(time.time())
+        query = self._api_query(path, version=5)
         sig = hmac.new(
             self._APP_SECRET.encode('ascii'),
-            query.encode('ascii'),
-            hashlib.sha1
-        ).hexdigest()
-        url = self._API_URL_TEMPLATE % (query, sig)
-        return sanitized_Request(
-            url, json.dumps(post_data).encode('utf-8')) if post_data else url
+            '{query}&t={timestamp}'.format(**locals()).encode('ascii'),
+            hashlib.sha1).hexdigest()
+        return timestamp, sig, self._API_URL_TEMPLATE % query
 
-    def _call_api(self, path, video_id, note, timestamp=None, post_data=None):
+    def _call_api(
+            self, path, video_id, note='Downloading JSON metadata', data=None, query=None, fatal=True):
+        if query is None:
+            timestamp, sig, url = self._sign_query(path)
+        else:
+            url = self._API_URL_TEMPLATE % self._api_query(path, version=4)
         resp = self._download_json(
-            self._prepare_call(path, timestamp, post_data), video_id, note,
-            headers={'x-viki-app-ver': self._APP_VERSION})
-
-        error = resp.get('error')
-        if error:
-            if error == 'invalid timestamp':
-                resp = self._download_json(
-                    self._prepare_call(path, int(resp['current_timestamp']), post_data),
-                    video_id, '%s (retry)' % note)
-                error = resp.get('error')
-            if error:
-                self._raise_error(resp['error'])
+            url, video_id, note, fatal=fatal, query=query,
+            data=json.dumps(data).encode('utf-8') if data else None,
+            headers=({'x-viki-app-ver': self._APP_VERSION} if data
+                     else self._stream_headers(timestamp, sig) if query is None
+                     else None), expected_status=400) or {}
 
+        self._raise_error(resp.get('error'), fatal)
         return resp
 
-    def _raise_error(self, error):
-        raise ExtractorError(
-            '%s returned error: %s' % (self.IE_NAME, error),
-            expected=True)
+    def _raise_error(self, error, fatal=True):
+        if error is None:
+            return
+        msg = '%s said: %s' % (self.IE_NAME, error)
+        if fatal:
+            raise ExtractorError(msg, expected=True)
+        else:
+            self.report_warning(msg)
 
     def _check_errors(self, data):
         for reason, status in (data.get('blocking') or {}).items():
@@ -90,9 +98,10 @@ class VikiBaseIE(InfoExtractor):
                 if reason == 'geo':
                     self.raise_geo_restricted(msg=message)
                 elif reason == 'paywall':
+                    if try_get(data, lambda x: x['paywallable']['tvod']):
+                        self._raise_error('This video is for rent only or TVOD (Transactional Video On demand)')
                     self.raise_login_required(message)
-                raise ExtractorError('%s said: %s' % (
-                    self.IE_NAME, message), expected=True)
+                self._raise_error(message)
 
     def _real_initialize(self):
         self._login()
@@ -102,35 +111,39 @@ class VikiBaseIE(InfoExtractor):
         if username is None:
             return
 
-        login_form = {
-            'login_id': username,
-            'password': password,
-        }
-
-        login = self._call_api(
-            'sessions.json', None,
-            'Logging in', post_data=login_form)
-
-        self._token = login.get('token')
+        self._token = self._call_api(
+            'sessions.json', None, 'Logging in', fatal=False,
+            data={'username': username, 'password': password}).get('token')
         if not self._token:
-            self.report_warning('Unable to get session token, login has probably failed')
+            self.report_warning('Login Failed: Unable to get session token')
 
     @staticmethod
-    def dict_selection(dict_obj, preferred_key, allow_fallback=True):
+    def dict_selection(dict_obj, preferred_key):
         if preferred_key in dict_obj:
-            return dict_obj.get(preferred_key)
-
-        if not allow_fallback:
-            return
-
-        filtered_dict = list(filter(None, [dict_obj.get(k) for k in dict_obj.keys()]))
-        return filtered_dict[0] if filtered_dict else None
+            return dict_obj[preferred_key]
+        return (list(filter(None, dict_obj.values())) or [None])[0]
 
 
 class VikiIE(VikiBaseIE):
     IE_NAME = 'viki'
     _VALID_URL = r'%s(?:videos|player)/(?P<id>[0-9]+v)' % VikiBaseIE._VALID_URL_BASE
     _TESTS = [{
+        'note': 'Free non-DRM video with storyboards in MPD',
+        'url': 'https://www.viki.com/videos/1175236v-choosing-spouse-by-lottery-episode-1',
+        'info_dict': {
+            'id': '1175236v',
+            'ext': 'mp4',
+            'title': 'Choosing Spouse by Lottery - Episode 1',
+            'timestamp': 1606463239,
+            'age_limit': 12,
+            'uploader': 'FCC',
+            'upload_date': '20201127',
+        },
+        'expected_warnings': ['Unknown MIME type image/jpeg in DASH manifest'],
+        'params': {
+            'format': 'bestvideo',
+        },
+    }, {
         'url': 'http://www.viki.com/videos/1023585v-heirs-episode-14',
         'info_dict': {
             'id': '1023585v',
@@ -146,7 +159,7 @@ class VikiIE(VikiBaseIE):
         'params': {
             'format': 'bestvideo',
         },
-        'skip': 'Blocked in the US',
+        'skip': 'Content is only available to Viki Pass Plus subscribers',
         'expected_warnings': ['Unknown MIME type image/jpeg in DASH manifest'],
     }, {
         # clip
@@ -178,11 +191,11 @@ class VikiIE(VikiBaseIE):
             'like_count': int,
             'age_limit': 13,
         },
-        'skip': 'Blocked in the US',
+        'skip': 'Page not found!',
     }, {
         # episode
         'url': 'http://www.viki.com/videos/44699v-boys-over-flowers-episode-1',
-        'md5': '0a53dc252e6e690feccd756861495a8c',
+        'md5': '670440c79f7109ca6564d4c7f24e3e81',
         'info_dict': {
             'id': '44699v',
             'ext': 'mp4',
@@ -193,7 +206,7 @@ class VikiIE(VikiBaseIE):
             'upload_date': '20100405',
             'uploader': 'group8',
             'like_count': int,
-            'age_limit': 13,
+            'age_limit': 15,
             'episode_number': 1,
         },
         'params': {
@@ -224,7 +237,7 @@ class VikiIE(VikiBaseIE):
     }, {
         # non-English description
         'url': 'http://www.viki.com/videos/158036v-love-in-magic',
-        'md5': '41faaba0de90483fb4848952af7c7d0d',
+        'md5': '78bf49fdaa51f9e7f9150262a9ef9bdf',
         'info_dict': {
             'id': '158036v',
             'ext': 'mp4',
@@ -232,8 +245,8 @@ class VikiIE(VikiBaseIE):
             'upload_date': '20111122',
             'timestamp': 1321985454,
             'description': 'md5:44b1e46619df3a072294645c770cef36',
-            'title': 'Love In Magic',
-            'age_limit': 13,
+            'title': 'Love in Magic',
+            'age_limit': 15,
         },
         'params': {
             'format': 'bestvideo',
@@ -244,45 +257,53 @@ class VikiIE(VikiBaseIE):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        resp = self._download_json(
-            'https://www.viki.com/api/videos/' + video_id,
-            video_id, 'Downloading video JSON', headers={
-                'x-client-user-agent': std_headers['User-Agent'],
-                'x-viki-app-ver': '3.0.0',
-            })
-        video = resp['video']
+        video = self._call_api('videos/{0}.json'.format(video_id), video_id, 'Downloading video JSON', query={})
 
         self._check_errors(video)
 
-        title = self.dict_selection(video.get('titles', {}), 'en', allow_fallback=False)
+        title = try_get(video, lambda x: x['titles']['en'], str)
         episode_number = int_or_none(video.get('number'))
         if not title:
             title = 'Episode %d' % episode_number if video.get('type') == 'episode' else video.get('id') or video_id
             container_titles = try_get(video, lambda x: x['container']['titles'], dict) or {}
             container_title = self.dict_selection(container_titles, 'en')
-            title = '%s - %s' % (container_title, title)
+            if container_title and title == video_id:
+                title = container_title
+            else:
+                title = '%s - %s' % (container_title, title)
+
+        resp = self._call_api(
+            'playback_streams/%s.json?drms=dt3&device_id=%s' % (video_id, self._DEVICE_ID),
+            video_id, 'Downloading video streams JSON')['main'][0]
+
+        mpd_url = resp['url']
+        # 720p is hidden in another MPD which can be found in the current manifest content
+        mpd_content = self._download_webpage(mpd_url, video_id, note='Downloading initial MPD manifest')
+        mpd_url = self._search_regex(
+            r'(?mi)<BaseURL>(http.+.mpd)', mpd_content, 'new manifest', default=mpd_url)
+        if 'mpdhd_high' not in mpd_url:
+            # Modify the URL to get 1080p
+            mpd_url = mpd_url.replace('mpdhd', 'mpdhd_high')
+        formats = self._extract_mpd_formats(mpd_url, video_id)
+        self._sort_formats(formats)
 
         description = self.dict_selection(video.get('descriptions', {}), 'en')
-
+        thumbnails = [{
+            'id': thumbnail_id,
+            'url': thumbnail['url'],
+        } for thumbnail_id, thumbnail in (video.get('images') or {}).items() if thumbnail.get('url')]
         like_count = int_or_none(try_get(video, lambda x: x['likes']['count']))
 
-        thumbnails = []
-        for thumbnail_id, thumbnail in (video.get('images') or {}).items():
-            thumbnails.append({
-                'id': thumbnail_id,
-                'url': thumbnail.get('url'),
-            })
+        stream_id = try_get(resp, lambda x: x['properties']['track']['stream_id'])
+        subtitles = dict((lang, [{
+            'ext': ext,
+            'url': self._API_URL_TEMPLATE % self._api_query(
+                'videos/{0}/auth_subtitles/{1}.{2}'.format(video_id, lang, ext), stream_id=stream_id)
+        } for ext in ('srt', 'vtt')]) for lang in (video.get('subtitle_completions') or {}).keys())
 
-        subtitles = {}
-        for subtitle_lang, _ in (video.get('subtitle_completions') or {}).items():
-            subtitles[subtitle_lang] = [{
-                'ext': subtitles_format,
-                'url': self._prepare_call(
-                    'videos/%s/subtitles/%s.%s' % (video_id, subtitle_lang, subtitles_format)),
-            } for subtitles_format in ('srt', 'vtt')]
-
-        result = {
+        return {
             'id': video_id,
+            'formats': formats,
             'title': title,
             'description': description,
             'duration': int_or_none(video.get('duration')),
@@ -296,79 +317,6 @@ class VikiIE(VikiBaseIE):
             'episode_number': episode_number,
         }
 
-        formats = []
-
-        def add_format(format_id, format_dict, protocol='http'):
-            # rtmps URLs does not seem to work
-            if protocol == 'rtmps':
-                return
-            format_url = format_dict.get('url')
-            if not format_url:
-                return
-            qs = compat_parse_qs(compat_urllib_parse_urlparse(format_url).query)
-            stream = qs.get('stream', [None])[0]
-            if stream:
-                format_url = base64.b64decode(stream).decode()
-            if format_id in ('m3u8', 'hls'):
-                m3u8_formats = self._extract_m3u8_formats(
-                    format_url, video_id, 'mp4',
-                    entry_protocol='m3u8_native',
-                    m3u8_id='m3u8-%s' % protocol, fatal=False)
-                # Despite CODECS metadata in m3u8 all video-only formats
-                # are actually video+audio
-                for f in m3u8_formats:
-                    if '_drm/index_' in f['url']:
-                        continue
-                    if f.get('acodec') == 'none' and f.get('vcodec') != 'none':
-                        f['acodec'] = None
-                    formats.append(f)
-            elif format_id in ('mpd', 'dash'):
-                formats.extend(self._extract_mpd_formats(
-                    format_url, video_id, 'mpd-%s' % protocol, fatal=False))
-            elif format_url.startswith('rtmp'):
-                mobj = re.search(
-                    r'^(?P<url>rtmp://[^/]+/(?P<app>.+?))/(?P<playpath>mp4:.+)$',
-                    format_url)
-                if not mobj:
-                    return
-                formats.append({
-                    'format_id': 'rtmp-%s' % format_id,
-                    'ext': 'flv',
-                    'url': mobj.group('url'),
-                    'play_path': mobj.group('playpath'),
-                    'app': mobj.group('app'),
-                    'page_url': url,
-                })
-            else:
-                formats.append({
-                    'url': format_url,
-                    'format_id': '%s-%s' % (format_id, protocol),
-                    'height': int_or_none(self._search_regex(
-                        r'^(\d+)[pP]$', format_id, 'height', default=None)),
-                })
-
-        for format_id, format_dict in (resp.get('streams') or {}).items():
-            add_format(format_id, format_dict)
-        if not formats:
-            streams = self._call_api(
-                'videos/%s/streams.json' % video_id, video_id,
-                'Downloading video streams JSON')
-
-            if 'external' in streams:
-                result.update({
-                    '_type': 'url_transparent',
-                    'url': streams['external']['url'],
-                })
-                return result
-
-            for format_id, stream_dict in streams.items():
-                for protocol, format_dict in stream_dict.items():
-                    add_format(format_id, format_dict, protocol)
-        self._sort_formats(formats)
-
-        result['formats'] = formats
-        return result
-
 
 class VikiChannelIE(VikiBaseIE):
     IE_NAME = 'viki:channel'
@@ -378,9 +326,9 @@ class VikiChannelIE(VikiBaseIE):
         'info_dict': {
             'id': '50c',
             'title': 'Boys Over Flowers',
-            'description': 'md5:804ce6e7837e1fd527ad2f25420f4d59',
+            'description': 'md5:f08b679c200e1a273c695fe9986f21d7',
         },
-        'playlist_mincount': 71,
+        'playlist_mincount': 51,
     }, {
         'url': 'http://www.viki.com/tv/1354c-poor-nastya-complete',
         'info_dict': {
@@ -401,33 +349,38 @@ class VikiChannelIE(VikiBaseIE):
         'only_matching': True,
     }]
 
-    _PER_PAGE = 25
+    _video_types = ('episodes', 'movies', 'clips', 'trailers')
+
+    def _entries(self, channel_id):
+        params = {
+            'app': self._APP, 'token': self._token, 'only_ids': 'true',
+            'direction': 'asc', 'sort': 'number', 'per_page': 30
+        }
+        video_types = self._video_types
+        for video_type in video_types:
+            if video_type not in self._video_types:
+                self.report_warning('Unknown video_type: ' + video_type)
+            page_num = 0
+            while True:
+                page_num += 1
+                params['page'] = page_num
+                res = self._call_api(
+                    'containers/{channel_id}/{video_type}.json'.format(**locals()), channel_id, query=params, fatal=False,
+                    note='Downloading %s JSON page %d' % (video_type.title(), page_num))
+
+                for video_id in res.get('response') or []:
+                    yield self.url_result('https://www.viki.com/videos/' + video_id, VikiIE.ie_key(), video_id)
+                if not res.get('more'):
+                    break
 
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
-        channel = self._call_api(
-            'containers/%s.json' % channel_id, channel_id,
-            'Downloading channel JSON')
+        channel = self._call_api('containers/%s.json' % channel_id, channel_id, 'Downloading channel JSON')
 
         self._check_errors(channel)
 
-        title = self.dict_selection(channel['titles'], 'en')
-
-        description = self.dict_selection(channel['descriptions'], 'en')
-
-        entries = []
-        for video_type in ('episodes', 'clips', 'movies'):
-            for page_num in itertools.count(1):
-                page = self._call_api(
-                    'containers/%s/%s.json?per_page=%d&sort=number&direction=asc&with_paging=true&page=%d'
-                    % (channel_id, video_type, self._PER_PAGE, page_num), channel_id,
-                    'Downloading %s JSON page #%d' % (video_type, page_num))
-                for video in page['response']:
-                    video_id = video['id']
-                    entries.append(self.url_result(
-                        'https://www.viki.com/videos/%s' % video_id, 'Viki'))
-                if not page['pagination']['next']:
-                    break
-
-        return self.playlist_result(entries, channel_id, title, description)
+        return self.playlist_result(
+            self._entries(channel_id), channel_id,
+            self.dict_selection(channel['titles'], 'en'),
+            self.dict_selection(channel['descriptions'], 'en'))

From b494824286f0ac2fc7313452b287fbbffe61ccbe Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 17 Jan 2022 13:11:11 +0000
Subject: [PATCH 447/860] Support Tele5 pages with Discovery Networks format
 instead of JWPlatform

---
 youtube_dl/extractor/tele5.py | 86 ++++++++++++++---------------------
 1 file changed, 35 insertions(+), 51 deletions(-)

diff --git a/youtube_dl/extractor/tele5.py b/youtube_dl/extractor/tele5.py
index 3e1a7a9e6..df02dfc47 100644
--- a/youtube_dl/extractor/tele5.py
+++ b/youtube_dl/extractor/tele5.py
@@ -1,19 +1,16 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
-from .common import InfoExtractor
-from .jwplatform import JWPlatformIE
-from .nexx import NexxIE
 from ..compat import compat_urlparse
 from ..utils import (
-    NO_DEFAULT,
-    smuggle_url,
+    ExtractorError,
+    extract_attributes,
 )
 
+from .dplay import DPlayIE
 
-class Tele5IE(InfoExtractor):
+
+class Tele5IE(DPlayIE):
     _VALID_URL = r'https?://(?:www\.)?tele5\.de/(?:[^/]+/)*(?P<id>[^/?#&]+)'
     _GEO_COUNTRIES = ['DE']
     _TESTS = [{
@@ -28,6 +25,7 @@ class Tele5IE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+        'skip': 'No longer available: "404 Seite nicht gefunden"',
     }, {
         # jwplatform, nexx unavailable
         'url': 'https://www.tele5.de/filme/ghoul-das-geheimnis-des-friedhofmonsters/',
@@ -42,7 +40,20 @@ class Tele5IE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
-        'add_ie': [JWPlatformIE.ie_key()],
+        'skip': 'No longer available, redirects to Filme page',
+    }, {
+        'url': 'https://tele5.de/mediathek/angel-of-mine/',
+        'info_dict': {
+            'id': '1252360',
+            'ext': 'mp4',
+            'upload_date': '20220109',
+            'timestamp': 1641762000,
+            'title': 'Angel of Mine',
+            'description': 'md5:a72546a175e1286eb3251843a52d1ad7',
+        },
+        'params': {
+            'format': 'bestvideo',
+        },
     }, {
         'url': 'https://www.tele5.de/kalkofes-mattscheibe/video-clips/politik-und-gesellschaft?ve_id=1551191',
         'only_matching': True,
@@ -64,45 +75,18 @@ class Tele5IE(InfoExtractor):
     }]
 
     def _real_extract(self, url):
-        qs = compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
-        video_id = (qs.get('vid') or qs.get('ve_id') or [None])[0]
-
-        NEXX_ID_RE = r'\d{6,}'
-        JWPLATFORM_ID_RE = r'[a-zA-Z0-9]{8}'
-
-        def nexx_result(nexx_id):
-            return self.url_result(
-                'https://api.nexx.cloud/v3/759/videos/byid/%s' % nexx_id,
-                ie=NexxIE.ie_key(), video_id=nexx_id)
-
-        nexx_id = jwplatform_id = None
-
-        if video_id:
-            if re.match(NEXX_ID_RE, video_id):
-                return nexx_result(video_id)
-            elif re.match(JWPLATFORM_ID_RE, video_id):
-                jwplatform_id = video_id
-
-        if not nexx_id:
-            display_id = self._match_id(url)
-            webpage = self._download_webpage(url, display_id)
-
-            def extract_id(pattern, name, default=NO_DEFAULT):
-                return self._html_search_regex(
-                    (r'id\s*=\s*["\']video-player["\'][^>]+data-id\s*=\s*["\'](%s)' % pattern,
-                     r'\s+id\s*=\s*["\']player_(%s)' % pattern,
-                     r'\bdata-id\s*=\s*["\'](%s)' % pattern), webpage, name,
-                    default=default)
-
-            nexx_id = extract_id(NEXX_ID_RE, 'nexx id', default=None)
-            if nexx_id:
-                return nexx_result(nexx_id)
-
-            if not jwplatform_id:
-                jwplatform_id = extract_id(JWPLATFORM_ID_RE, 'jwplatform id')
-
-        return self.url_result(
-            smuggle_url(
-                'jwplatform:%s' % jwplatform_id,
-                {'geo_countries': self._GEO_COUNTRIES}),
-            ie=JWPlatformIE.ie_key(), video_id=jwplatform_id)
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        player_element = self._search_regex(r'(<hyoga-player\b[^>]+?>)', webpage, 'video player')
+        player_info = extract_attributes(player_element)
+        asset_id, country, realm = (player_info[x] for x in ('assetid', 'locale', 'realm', ))
+        endpoint = compat_urlparse.urlparse(player_info['endpoint']).hostname
+        source_type = player_info.get('sourcetype')
+        if source_type:
+            endpoint = '%s-%s' % (source_type, endpoint)
+        try:
+            return self._get_disco_api_info(url, asset_id, endpoint, realm, country)
+        except ExtractorError as e:
+            if getattr(e, 'message', '') == 'Missing deviceId in context':
+                raise ExtractorError('DRM protected', cause=e, expected=True)
+            raise

From 4186e817772d49d6f66b07c5ac8c248f026a6446 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 17 Jan 2022 03:13:37 +0000
Subject: [PATCH 448/860] NDR: improve extraction of NDR id, description, etc
 with current page formats

---
 youtube_dl/extractor/ndr.py | 45 +++++++++++++++++++++++++++----------
 1 file changed, 33 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index ddd828d92..a0d553f00 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -4,8 +4,10 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+from ..compat import compat_urllib_parse_urlparse
 from ..utils import (
     determine_ext,
+    ExtractorError,
     int_or_none,
     merge_dicts,
     parse_iso8601,
@@ -20,13 +22,13 @@ class NDRBaseIE(InfoExtractor):
         mobj = re.match(self._VALID_URL, url)
         display_id = next(group for group in mobj.groups() if group)
         webpage = self._download_webpage(url, display_id)
-        return self._extract_embed(webpage, display_id)
+        return self._extract_embed(webpage, display_id, url)
 
 
 class NDRIE(NDRBaseIE):
     IE_NAME = 'ndr'
     IE_DESC = 'NDR.de - Norddeutscher Rundfunk'
-    _VALID_URL = r'https?://(?:www\.)?ndr\.de/(?:[^/]+/)*(?P<id>[^/?#]+),[\da-z]+\.html'
+    _VALID_URL = r'https?://(?:\w+\.)?ndr\.de/(?:[^/]+/)*(?P<id>[^/?#]+),[\da-z]+\.html'
     _TESTS = [{
         # httpVideo, same content id
         'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',
@@ -109,19 +111,38 @@ class NDRIE(NDRBaseIE):
         'only_matching': True,
     }]
 
-    def _extract_embed(self, webpage, display_id):
-        embed_url = self._html_search_meta(
-            'embedURL', webpage, 'embed URL',
-            default=None) or self._search_regex(
-            r'\bembedUrl["\']\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
-            'embed URL', group='url')
+    def _extract_embed(self, webpage, display_id, url):
+        embed_url = (
+            self._html_search_meta(
+                'embedURL', webpage, 'embed URL',
+                default=None)
+            or self._search_regex(
+                r'\bembedUrl["\']\s*:\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
+                'embed URL', group='url', default=None)
+            or self._search_regex(
+                r'\bvar\s*sophoraID\s*=\s*(["\'])(?P<url>(?:(?!\1).)+)\1', webpage,
+                'embed URL', group='url', default=''))
+        # some more work needed if we only found sophoraID
+        if re.match(r'^[a-z]+\d+$', embed_url):
+            # get the initial part of the url path,. eg /panorama/archiv/2022/
+            parsed_url = compat_urllib_parse_urlparse(url)
+            path = self._search_regex(r'(.+/)%s' % display_id, parsed_url.path or '', 'embed URL', default='')
+            # find tell-tale image with the actual ID
+            ndr_id = self._search_regex(r'%s([a-z]+\d+)(?!\.)\b' % (path, ), webpage, 'embed URL', default=None)
+            # or try to use special knowledge!
+            NDR_INFO_URL_TPL = 'https://www.ndr.de/info/%s-player.html'
+            embed_url = 'ndr:%s' % (ndr_id, ) if ndr_id else NDR_INFO_URL_TPL % (embed_url, )
+        if not embed_url:
+            raise ExtractorError('Unable to extract embedUrl')
+
         description = self._search_regex(
             r'<p[^>]+itemprop="description">([^<]+)</p>',
             webpage, 'description', default=None) or self._og_search_description(webpage)
         timestamp = parse_iso8601(
             self._search_regex(
-                r'<span[^>]+itemprop="(?:datePublished|uploadDate)"[^>]+content="([^"]+)"',
-                webpage, 'upload date', default=None))
+                (r'<span[^>]+itemprop="(?:datePublished|uploadDate)"[^>]+content="(?P<cont>[^"]+)"',
+                 r'\bvar\s*pdt\s*=\s*(?P<q>["\'])(?P<cont>(?:(?!(?P=q)).)+)(?P=q)', ),
+                webpage, 'upload date', group='cont', default=None))
         info = self._search_json_ld(webpage, display_id, default={})
         return merge_dicts({
             '_type': 'url_transparent',
@@ -179,7 +200,7 @@ class NJoyIE(NDRBaseIE):
         video_id = self._search_regex(
             r'<iframe[^>]+id="pp_([\da-z]+)"', webpage, 'embed id')
         description = self._search_regex(
-            r'<div[^>]+class="subline"[^>]*>[^<]+</div>\s*<p>([^<]+)</p>',
+                r'<div[^>]+class="subline"[^>]*>[^<]+</div>\s*<p>([^<]+)</p>',
             webpage, 'description', fatal=False)
         return {
             '_type': 'url_transparent',
@@ -291,7 +312,7 @@ class NDREmbedBaseIE(InfoExtractor):
 
 class NDREmbedIE(NDREmbedBaseIE):
     IE_NAME = 'ndr:embed'
-    _VALID_URL = r'https?://(?:www\.)?ndr\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:player|externalPlayer)\.html'
+    _VALID_URL = r'https?://(?:www\.)?ndr\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:(?:ard)?player|externalPlayer)\.html'
     _TESTS = [{
         'url': 'http://www.ndr.de/fernsehen/sendungen/ndr_aktuell/ndraktuell28488-player.html',
         'md5': '8b9306142fe65bbdefb5ce24edb6b0a9',

From f0a05a55c2ee512880546c056cfbec5ad3399798 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 17 Jan 2022 03:22:32 +0000
Subject: [PATCH 449/860] NJoy: improve extraction of NDR id, description, etc
 with current page formats

---
 youtube_dl/extractor/ndr.py | 15 +++++++++++----
 1 file changed, 11 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index a0d553f00..0a723e3b0 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -196,18 +196,25 @@ class NJoyIE(NDRBaseIE):
         'only_matching': True,
     }]
 
-    def _extract_embed(self, webpage, display_id):
+    def _extract_embed(self, webpage, display_id, url=None):
+        # find tell-tale URL with the actual ID, or ...
         video_id = self._search_regex(
-            r'<iframe[^>]+id="pp_([\da-z]+)"', webpage, 'embed id')
-        description = self._search_regex(
+            (r'''\bsrc\s*=\s*(?:"|')?(?:/\w+)+/([a-z]+\d+)(?!\.)\b''',
+             r'<iframe[^>]+id="pp_([\da-z]+)"', ),
+            webpage, 'NDR id', default=None)
+
+        description = (
+            self._html_search_meta('description', webpage)
+            or self._search_regex(
                 r'<div[^>]+class="subline"[^>]*>[^<]+</div>\s*<p>([^<]+)</p>',
-            webpage, 'description', fatal=False)
+                webpage, 'description', fatal=False))
         return {
             '_type': 'url_transparent',
             'ie_key': 'NDREmbedBase',
             'url': 'ndr:%s' % video_id,
             'display_id': display_id,
             'description': description,
+            'title': display_id.replace('-', ' ').strip(),
         }
 
 

From 39a98b09a2acf50dc64bc41185be723b98e740b9 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 17 Jan 2022 03:29:43 +0000
Subject: [PATCH 450/860] Fix NDR, NJoy tests

---
 youtube_dl/extractor/ndr.py | 41 ++++++++++++++++++++++---------------
 1 file changed, 25 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 0a723e3b0..1996d4f96 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -40,13 +40,14 @@ class NDRIE(NDRBaseIE):
             'title': 'Party, Pötte und Parade',
             'description': 'md5:ad14f9d2f91d3040b6930c697e5f6b4c',
             'uploader': 'ndrtv',
-            'timestamp': 1431108900,
+            'timestamp': 1431255671,
             'upload_date': '20150510',
             'duration': 3498,
         },
         'params': {
             'skip_download': True,
         },
+        'expected_warnings': ['Unable to download f4m manifest'],
     }, {
         # httpVideo, different content id
         'url': 'http://www.ndr.de/sport/fussball/40-Osnabrueck-spielt-sich-in-einen-Rausch,osna270.html',
@@ -65,6 +66,7 @@ class NDRIE(NDRBaseIE):
         'params': {
             'skip_download': True,
         },
+        'skip': 'No longer available',
     }, {
         # httpAudio, same content id
         'url': 'http://www.ndr.de/info/La-Valette-entgeht-der-Hinrichtung,audio51535.html',
@@ -76,8 +78,8 @@ class NDRIE(NDRBaseIE):
             'title': 'La Valette entgeht der Hinrichtung',
             'description': 'md5:22f9541913a40fe50091d5cdd7c9f536',
             'uploader': 'ndrinfo',
-            'timestamp': 1290626100,
-            'upload_date': '20140729',
+            'timestamp': 1631711863,
+            'upload_date': '20210915',
             'duration': 884,
         },
         'params': {
@@ -91,9 +93,10 @@ class NDRIE(NDRBaseIE):
             'display_id': 'extra-3-Satiremagazin-mit-Christian-Ehring',
             'ext': 'mp4',
             'title': 'Extra 3 vom 11.11.2020 mit Christian Ehring',
-            'description': 'md5:42ee53990a715eaaf4dc7f13a3bd56c6',
+            'description': 'md5:700f6de264010585012a72f97b0ac0c9',
             'uploader': 'ndrtv',
-            'upload_date': '20201113',
+            'upload_date': '20201207',
+            'timestamp': 1614349457,
             'duration': 1749,
             'subtitles': {
                 'de': [{
@@ -174,19 +177,19 @@ class NJoyIE(NDRBaseIE):
         'params': {
             'skip_download': True,
         },
+        'skip': 'No longer available',
     }, {
         # httpVideo, different content id
         'url': 'http://www.n-joy.de/musik/Das-frueheste-DJ-Set-des-Nordens-live-mit-Felix-Jaehn-,felixjaehn168.html',
         'md5': '417660fffa90e6df2fda19f1b40a64d8',
         'info_dict': {
-            'id': 'dockville882',
+            'id': 'livestream283',
             'display_id': 'Das-frueheste-DJ-Set-des-Nordens-live-mit-Felix-Jaehn-',
-            'ext': 'mp4',
-            'title': '"Ich hab noch nie" mit Felix Jaehn',
-            'description': 'md5:85dd312d53be1b99e1f998a16452a2f3',
+            'ext': 'mp3',
+            'title': 'Das frueheste DJ Set des Nordens live mit Felix Jaehn',
+            'description': 'md5:681698f527b8601e511e7b79edde7d2c',
             'uploader': 'njoy',
-            'upload_date': '20150822',
-            'duration': 211,
+            'upload_date': '20210830',
         },
         'params': {
             'skip_download': True,
@@ -332,6 +335,7 @@ class NDREmbedIE(NDREmbedBaseIE):
             'upload_date': '20150907',
             'duration': 132,
         },
+        'skip': 'No longer available',
     }, {
         'url': 'http://www.ndr.de/ndr2/events/soundcheck/soundcheck3366-player.html',
         'md5': '002085c44bae38802d94ae5802a36e78',
@@ -347,6 +351,7 @@ class NDREmbedIE(NDREmbedBaseIE):
         'params': {
             'skip_download': True,
         },
+        'skip': 'No longer available',
     }, {
         'url': 'http://www.ndr.de/info/audio51535-player.html',
         'md5': 'bb3cd38e24fbcc866d13b50ca59307b8',
@@ -356,7 +361,7 @@ class NDREmbedIE(NDREmbedBaseIE):
             'title': 'La Valette entgeht der Hinrichtung',
             'is_live': False,
             'uploader': 'ndrinfo',
-            'upload_date': '20140729',
+            'upload_date': '20210915',
             'duration': 884,
         },
         'params': {
@@ -377,15 +382,17 @@ class NDREmbedIE(NDREmbedBaseIE):
         'params': {
             'skip_download': True,
         },
+        'skip': 'No longer available',
     }, {
         # httpVideoLive
         'url': 'http://www.ndr.de/fernsehen/livestream/livestream217-externalPlayer.html',
         'info_dict': {
             'id': 'livestream217',
-            'ext': 'flv',
+            'ext': 'mp4',
             'title': r're:^NDR Fernsehen Niedersachsen \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
             'is_live': True,
-            'upload_date': '20150910',
+            'upload_date': '20210409',
+            'uploader': 'ndrtv',
         },
         'params': {
             'skip_download': True,
@@ -423,9 +430,10 @@ class NJoyEmbedIE(NDREmbedBaseIE):
             'ext': 'mp4',
             'title': 'Zehn Jahre Reeperbahn Festival - die Doku',
             'is_live': False,
-            'upload_date': '20150807',
+            'upload_date': '20200826',
             'duration': 1011,
         },
+        'expected_warnings': ['Unable to download f4m manifest'],
     }, {
         # httpAudio
         'url': 'http://www.n-joy.de/news_wissen/stefanrichter100-player_image-d5e938b1-f21a-4b9a-86b8-aaba8bca3a13_theme-n-joy.html',
@@ -442,6 +450,7 @@ class NJoyEmbedIE(NDREmbedBaseIE):
         'params': {
             'skip_download': True,
         },
+        'skip': 'No longer available',
     }, {
         # httpAudioLive, no explicit ext
         'url': 'http://www.n-joy.de/news_wissen/webradioweltweit100-player_image-3fec0484-2244-4565-8fb8-ed25fd28b173_theme-n-joy.html',
@@ -451,7 +460,7 @@ class NJoyEmbedIE(NDREmbedBaseIE):
             'title': r're:^N-JOY Weltweit \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
             'is_live': True,
             'uploader': 'njoy',
-            'upload_date': '20150810',
+            'upload_date': '20210830',
         },
         'params': {
             'skip_download': True,

From 01824d275bfa7efbaca274b38c1ddc2b03f12f5d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 19 Jan 2022 13:24:33 +0000
Subject: [PATCH 451/860] Additional tweaks: allow any .ndr.de, simplify quote
 match

---
 youtube_dl/extractor/ndr.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/ndr.py b/youtube_dl/extractor/ndr.py
index 1996d4f96..26627f8b0 100644
--- a/youtube_dl/extractor/ndr.py
+++ b/youtube_dl/extractor/ndr.py
@@ -28,7 +28,7 @@ class NDRBaseIE(InfoExtractor):
 class NDRIE(NDRBaseIE):
     IE_NAME = 'ndr'
     IE_DESC = 'NDR.de - Norddeutscher Rundfunk'
-    _VALID_URL = r'https?://(?:\w+\.)?ndr\.de/(?:[^/]+/)*(?P<id>[^/?#]+),[\da-z]+\.html'
+    _VALID_URL = r'https?://(?:\w+\.)*ndr\.de/(?:[^/]+/)*(?P<id>[^/?#]+),[\da-z]+\.html'
     _TESTS = [{
         # httpVideo, same content id
         'url': 'http://www.ndr.de/fernsehen/Party-Poette-und-Parade,hafengeburtstag988.html',
@@ -202,7 +202,7 @@ class NJoyIE(NDRBaseIE):
     def _extract_embed(self, webpage, display_id, url=None):
         # find tell-tale URL with the actual ID, or ...
         video_id = self._search_regex(
-            (r'''\bsrc\s*=\s*(?:"|')?(?:/\w+)+/([a-z]+\d+)(?!\.)\b''',
+            (r'''\bsrc\s*=\s*["']?(?:/\w+)+/([a-z]+\d+)(?!\.)\b''',
              r'<iframe[^>]+id="pp_([\da-z]+)"', ),
             webpage, 'NDR id', default=None)
 
@@ -322,7 +322,7 @@ class NDREmbedBaseIE(InfoExtractor):
 
 class NDREmbedIE(NDREmbedBaseIE):
     IE_NAME = 'ndr:embed'
-    _VALID_URL = r'https?://(?:www\.)?ndr\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:(?:ard)?player|externalPlayer)\.html'
+    _VALID_URL = r'https?://(?:\w+\.)*ndr\.de/(?:[^/]+/)*(?P<id>[\da-z]+)-(?:(?:ard)?player|externalPlayer)\.html'
     _TESTS = [{
         'url': 'http://www.ndr.de/fernsehen/sendungen/ndr_aktuell/ndraktuell28488-player.html',
         'md5': '8b9306142fe65bbdefb5ce24edb6b0a9',

From 5197336de6ee2d18c37732f3f7c6532c8899ec29 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 14 Jan 2022 20:14:14 +0000
Subject: [PATCH 452/860] Support more deeply nested ptmd_path with test,
 update tests

---
 youtube_dl/extractor/zdf.py | 28 ++++++++++++++++++++++++----
 1 file changed, 24 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 4dd56f66d..3d39bb33a 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -7,6 +7,7 @@ from .common import InfoExtractor
 from ..compat import compat_str
 from ..utils import (
     determine_ext,
+    ExtractorError,
     float_or_none,
     int_or_none,
     merge_dicts,
@@ -145,6 +146,7 @@ class ZDFIE(ZDFBaseIE):
             'timestamp': 1613948400,
             'upload_date': '20210221',
         },
+        'skip': 'No longer available: "Diese Seite wurde leider nicht gefunden"',
     }, {
         # Same as https://www.3sat.de/film/ab-18/10-wochen-sommer-108.html
         'url': 'https://www.zdf.de/dokumentation/ab-18/10-wochen-sommer-102.html',
@@ -158,6 +160,7 @@ class ZDFIE(ZDFBaseIE):
             'timestamp': 1608604200,
             'upload_date': '20201222',
         },
+        'skip': 'No longer available: "Diese Seite wurde leider nicht gefunden"',
     }, {
         'url': 'https://www.zdf.de/dokumentation/terra-x/die-magie-der-farben-von-koenigspurpur-und-jeansblau-100.html',
         'info_dict': {
@@ -190,6 +193,17 @@ class ZDFIE(ZDFBaseIE):
     }, {
         'url': 'https://www.zdf.de/dokumentation/planet-e/planet-e-uebersichtsseite-weitere-dokumentationen-von-planet-e-100.html',
         'only_matching': True,
+    }, {
+        'url': 'https://www.zdf.de/arte/todliche-flucht/page-video-artede-toedliche-flucht-16-100.html',
+        'info_dict': {
+            'id': 'video_artede_083871-001-A',
+            'ext': 'mp4',
+            'title': 'Tödliche Flucht (1/6)',
+            'description': 'md5:e34f96a9a5f8abd839ccfcebad3d5315',
+            'duration': 3193.0,
+            'timestamp': 1641355200,
+            'upload_date': '20220105',
+        },
     }]
 
     def _extract_entry(self, url, player, content, video_id):
@@ -197,12 +211,18 @@ class ZDFIE(ZDFBaseIE):
 
         t = content['mainVideoContent']['http://zdf.de/rels/target']
 
-        ptmd_path = t.get('http://zdf.de/rels/streams/ptmd')
+        def get_ptmd_path(d):
+            return (
+                d.get('http://zdf.de/rels/streams/ptmd')
+                or d.get('http://zdf.de/rels/streams/ptmd-template',
+                         '').replace('{playerId}', 'ngplayer_2_4'))
+
+        ptmd_path = get_ptmd_path(try_get(t, lambda x: x['streams']['default'], dict) or {})
+        if not ptmd_path:
+            ptmd_path = get_ptmd_path(t)
 
         if not ptmd_path:
-            ptmd_path = t[
-                'http://zdf.de/rels/streams/ptmd-template'].replace(
-                '{playerId}', 'ngplayer_2_4')
+            raise ExtractorError('Could not extract ptmd_path')
 
         info = self._extract_ptmd(
             urljoin(url, ptmd_path), video_id, player['apiToken'], url)

From 5cb4833f408745135d1b0e178b9a2545a899f2ac Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 13 Jan 2022 19:38:08 +0000
Subject: [PATCH 453/860] Update URPlayIE extractor for Next.js page format,
 with subtitles

---
 youtube_dl/extractor/urplay.py | 52 ++++++++++++++++++++++++++++++----
 1 file changed, 46 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/urplay.py b/youtube_dl/extractor/urplay.py
index d6c79147e..abd2bee84 100644
--- a/youtube_dl/extractor/urplay.py
+++ b/youtube_dl/extractor/urplay.py
@@ -4,7 +4,11 @@ from __future__ import unicode_literals
 from .common import InfoExtractor
 from ..utils import (
     dict_get,
+    ExtractorError,
     int_or_none,
+    ISO639Utils,
+    parse_age_limit,
+    try_get,
     unified_timestamp,
 )
 
@@ -23,9 +27,10 @@ class URPlayIE(InfoExtractor):
             'upload_date': '20171214',
             'series': 'UR Samtiden - Livet, universum och rymdens märkliga musik',
             'duration': 2269,
-            'categories': ['Kultur & historia'],
+            'categories': ['Vetenskap & teknik'],
             'tags': ['Kritiskt tänkande', 'Vetenskap', 'Vetenskaplig verksamhet'],
             'episode': 'Om vetenskap, kritiskt tänkande och motstånd',
+            'age_limit': 15,
         },
     }, {
         'url': 'https://urskola.se/Produkter/190031-Tripp-Trapp-Trad-Sovkudde',
@@ -50,11 +55,19 @@ class URPlayIE(InfoExtractor):
         video_id = self._match_id(url)
         url = url.replace('skola.se/Produkter', 'play.se/program')
         webpage = self._download_webpage(url, video_id)
-        vid = int(video_id)
-        accessible_episodes = self._parse_json(self._html_search_regex(
-            r'data-react-class="routes/Product/components/ProgramContainer/ProgramContainer"[^>]+data-react-props="({.+?})"',
-            webpage, 'urplayer data'), video_id)['accessibleEpisodes']
-        urplayer_data = next(e for e in accessible_episodes if e.get('id') == vid)
+        urplayer_data = self._search_regex(
+            r'(?s)\bid\s*=\s*"__NEXT_DATA__"[^>]*>\s*({.+?})\s*</script',
+            webpage, 'urplayer next data', fatal=False) or {}
+        if urplayer_data:
+            urplayer_data = self._parse_json(urplayer_data, video_id, fatal=False)
+            urplayer_data = try_get(urplayer_data, lambda x: x['props']['pageProps']['program'], dict)
+            if not urplayer_data:
+                raise ExtractorError('Unable to parse __NEXT_DATA__')
+        else:
+            accessible_episodes = self._parse_json(self._html_search_regex(
+                r'data-react-class="routes/Product/components/ProgramContainer/ProgramContainer"[^>]+data-react-props="({.+?})"',
+                webpage, 'urplayer data'), video_id)['accessibleEpisodes']
+            urplayer_data = next(e for e in accessible_episodes if e.get('id') == int_or_none(video_id))
         episode = urplayer_data['title']
         raw_streaming_info = urplayer_data['streamingInfo']['raw']
         host = self._download_json(
@@ -72,6 +85,30 @@ class URPlayIE(InfoExtractor):
                     video_id, skip_protocols=['f4m', 'rtmp', 'rtsp']))
         self._sort_formats(formats)
 
+        subtitles = {}
+
+        def parse_lang_code(code):
+            "3-character language code or None (utils candidate)"
+            if code is None:
+                return
+            lang = code.lower()
+            if not ISO639Utils.long2short(lang):
+                lang = ISO639Utils.short2long(lang)
+            return lang or None
+
+        for k, v in (urplayer_data['streamingInfo'].get('sweComplete') or {}).items():
+            if (k in ('sd', 'hd') or not isinstance(v, dict)):
+                continue
+            lang, sttl_url = (v.get(kk) for kk in ('language', 'location', ))
+            if not sttl_url:
+                continue
+            lang = parse_lang_code(lang)
+            if not lang:
+                continue
+            sttl = subtitles.get(lang) or []
+            sttl.append({'ext': k, 'url': sttl_url, })
+            subtitles[lang] = sttl
+
         image = urplayer_data.get('image') or {}
         thumbnails = []
         for k, v in image.items():
@@ -104,4 +141,7 @@ class URPlayIE(InfoExtractor):
             'season': series.get('label'),
             'episode': episode,
             'episode_number': int_or_none(urplayer_data.get('episodeNumber')),
+            'age_limit': parse_age_limit(min(try_get(a, lambda x: x['from'], int) or 0
+                                             for a in urplayer_data.get('ageRanges', []))),
+            'subtitles': subtitles,
         }

From 568c7005d513d0398c20b9e88eb9838c68651fc2 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 25 Jan 2022 12:59:31 +0000
Subject: [PATCH 454/860] Fix WDRMaus; extend URL matching for other Maus
 pages; improve ID extraction

---
 youtube_dl/extractor/wdr.py | 39 +++++++++++++++++++++++++++----------
 1 file changed, 29 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index 2903d189e..a5488f3fd 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -10,6 +10,7 @@ from ..compat import (
 )
 from ..utils import (
     determine_ext,
+    dict_get,
     ExtractorError,
     js_to_json,
     strip_jsonp,
@@ -22,9 +23,10 @@ from ..utils import (
 
 
 class WDRIE(InfoExtractor):
-    _VALID_URL = r'https?://deviceids-medp\.wdr\.de/ondemand/\d+/(?P<id>\d+)\.js'
+    __API_URL_TPL = '//deviceids-medp.wdr.de/ondemand/%s/%s'
+    _VALID_URL = (r'(?:https?:' + __API_URL_TPL) % (r'\d+', r'(?=\d+\.js)|wdr:)(?P<id>\d{6,})')
     _GEO_COUNTRIES = ['DE']
-    _TEST = {
+    _TESTS = [{
         'url': 'http://deviceids-medp.wdr.de/ondemand/155/1557833.js',
         'info_dict': {
             'id': 'mdb-1557833',
@@ -32,11 +34,20 @@ class WDRIE(InfoExtractor):
             'title': 'Biathlon-Staffel verpasst Podest bei Olympia-Generalprobe',
             'upload_date': '20180112',
         },
-    }
+    },
+    ]
+
+    def _asset_url(self, wdr_id):
+        id_len = max(len(wdr_id), 5)
+        return ''.join(('https:', self.__API_URL_TPL % (wdr_id[:id_len - 4], wdr_id, ), '.js'))
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
+        if url.startswith('wdr:'):
+            video_id = url[4:]
+            url = self._asset_url(video_id)
+
         metadata = self._download_json(
             url, video_id, transform_source=strip_jsonp)
 
@@ -115,10 +126,10 @@ class WDRIE(InfoExtractor):
         }
 
 
-class WDRPageIE(InfoExtractor):
-    _CURRENT_MAUS_URL = r'https?://(?:www\.)wdrmaus.de/(?:[^/]+/){1,2}[^/?#]+\.php5'
+class WDRPageIE(WDRIE):
+    _MAUS_REGEX = r'https?://(?:www\.)wdrmaus.de/(?:[^/]+/)*?(?P<maus_id>[^/?#.]+)(?:/?|/index\.php5|\.php5)$'
     _PAGE_REGEX = r'/(?:mediathek/)?(?:[^/]+/)*(?P<display_id>[^/]+)\.html'
-    _VALID_URL = r'https?://(?:www\d?\.)?(?:(?:kinder\.)?wdr\d?|sportschau)\.de' + _PAGE_REGEX + '|' + _CURRENT_MAUS_URL
+    _VALID_URL = r'https?://(?:www\d?\.)?(?:(?:kinder\.)?wdr\d?|sportschau)\.de' + _PAGE_REGEX + '|' + _MAUS_REGEX
 
     _TESTS = [
         {
@@ -180,12 +191,12 @@ class WDRPageIE(InfoExtractor):
         {
             'url': 'http://www.wdrmaus.de/aktuelle-sendung/index.php5',
             'info_dict': {
-                'id': 'mdb-1552552',
+                'id': 'mdb-2627637',
                 'ext': 'mp4',
                 'upload_date': 're:^[0-9]{8}$',
-                'title': 're:^Die Sendung mit der Maus vom [0-9.]{10}$',
+                'title': 're:^Die Sendung (?:mit der Maus )?vom [0-9.]{10}$',
             },
-            'skip': 'The id changes from week to week because of the new episode'
+            # 'skip': 'The id changes from week to week because of the new episode'
         },
         {
             'url': 'http://www.wdrmaus.de/filme/sachgeschichten/achterbahn.php5',
@@ -234,7 +245,7 @@ class WDRPageIE(InfoExtractor):
 
     def _real_extract(self, url):
         mobj = re.match(self._VALID_URL, url)
-        display_id = mobj.group('display_id')
+        display_id = dict_get(mobj.groupdict(), ('display_id', 'maus_id'), 'wdrmaus')
         webpage = self._download_webpage(url, display_id)
 
         entries = []
@@ -260,6 +271,14 @@ class WDRPageIE(InfoExtractor):
             jsonp_url = try_get(
                 media_link_obj, lambda x: x['mediaObj']['url'], compat_str)
             if jsonp_url:
+                # metadata, or player JS with ['ref'] giving WDR id, or just media, perhaps
+                clip_id = media_link_obj['mediaObj'].get('ref')
+                if jsonp_url.endswith('.assetjsonp'):
+                    asset = self._download_json(
+                        jsonp_url, display_id, fatal=False, transform_source=strip_jsonp)
+                    clip_id = try_get(asset, lambda x: x['trackerData']['trackerClipId'], compat_str)
+                if clip_id:
+                    jsonp_url = self._asset_url(clip_id[4:])
                 entries.append(self.url_result(jsonp_url, ie=WDRIE.ie_key()))
 
         # Playlist (e.g. https://www1.wdr.de/mediathek/video/sendungen/aktuelle-stunde/aktuelle-stunde-120.html)

From 96423449659131ed8e7bfaa7f791466c3f8f2db1 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 25 Jan 2022 13:04:04 +0000
Subject: [PATCH 455/860] Fix tests for working IEs; disable obsolete WDRMobile

---
 youtube_dl/extractor/wdr.py | 25 +++++++++++++------------
 1 file changed, 13 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/wdr.py b/youtube_dl/extractor/wdr.py
index a5488f3fd..10db73148 100644
--- a/youtube_dl/extractor/wdr.py
+++ b/youtube_dl/extractor/wdr.py
@@ -170,11 +170,11 @@ class WDRPageIE(WDRIE):
         {
             'url': 'http://www1.wdr.de/mediathek/video/live/index.html',
             'info_dict': {
-                'id': 'mdb-1406149',
+                'id': 'mdb-2296252',
                 'ext': 'mp4',
-                'title': r're:^WDR Fernsehen im Livestream \(nur in Deutschland erreichbar\) [0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
+                'title': r're:^WDR Fernsehen im Livestream (?:\(nur in Deutschland erreichbar\) )?[0-9]{4}-[0-9]{2}-[0-9]{2} [0-9]{2}:[0-9]{2}$',
                 'alt_title': 'WDR Fernsehen Live',
-                'upload_date': '20150101',
+                'upload_date': '20201112',
                 'is_live': True,
             },
             'params': {
@@ -183,7 +183,7 @@ class WDRPageIE(WDRIE):
         },
         {
             'url': 'http://www1.wdr.de/mediathek/video/sendungen/aktuelle-stunde/aktuelle-stunde-120.html',
-            'playlist_mincount': 7,
+            'playlist_mincount': 6,
             'info_dict': {
                 'id': 'aktuelle-stunde-120',
             },
@@ -196,7 +196,7 @@ class WDRPageIE(WDRIE):
                 'upload_date': 're:^[0-9]{8}$',
                 'title': 're:^Die Sendung (?:mit der Maus )?vom [0-9.]{10}$',
             },
-            # 'skip': 'The id changes from week to week because of the new episode'
+            'skip': 'The id changes from week to week because of the new episode'
         },
         {
             'url': 'http://www.wdrmaus.de/filme/sachgeschichten/achterbahn.php5',
@@ -207,6 +207,7 @@ class WDRPageIE(WDRIE):
                 'upload_date': '20130919',
                 'title': 'Sachgeschichte - Achterbahn ',
             },
+            'skip': 'HTTP Error 404: Not Found',
         },
         {
             'url': 'http://www1.wdr.de/radio/player/radioplayer116~_layout-popupVersion.html',
@@ -232,6 +233,7 @@ class WDRPageIE(WDRIE):
             'params': {
                 'skip_download': True,
             },
+            'skip': 'HTTP Error 404: Not Found',
         },
         {
             'url': 'http://www.sportschau.de/handballem2018/audio-vorschau---die-handball-em-startet-mit-grossem-favoritenfeld-100.html',
@@ -298,16 +300,14 @@ class WDRPageIE(WDRIE):
 class WDRElefantIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)wdrmaus\.de/elefantenseite/#(?P<id>.+)'
     _TEST = {
-        'url': 'http://www.wdrmaus.de/elefantenseite/#folge_ostern_2015',
+        'url': 'http://www.wdrmaus.de/elefantenseite/#elefantenkino_wippe',
+        # adaptive stream: unstable file MD5
         'info_dict': {
-            'title': 'Folge Oster-Spezial 2015',
-            'id': 'mdb-1088195',
+            'title': 'Wippe',
+            'id': 'mdb-1198320',
             'ext': 'mp4',
             'age_limit': None,
-            'upload_date': '20150406'
-        },
-        'params': {
-            'skip_download': True,
+            'upload_date': '20071003'
         },
     }
 
@@ -342,6 +342,7 @@ class WDRMobileIE(InfoExtractor):
         /[0-9]+/[0-9]+/
         (?P<id>[0-9]+)_(?P<title>[0-9]+)'''
     IE_NAME = 'wdr:mobile'
+    _WORKING = False  # no such domain
     _TEST = {
         'url': 'http://mobile-ondemand.wdr.de/CMS2010/mdb/ondemand/weltweit/fsk0/42/421735/421735_4283021.mp4',
         'info_dict': {

From 23ad6402a6966dd09e4c854f32c33f69be1a064e Mon Sep 17 00:00:00 2001
From: Chris Rose <offline@offby1.net>
Date: Fri, 26 Nov 2021 08:08:17 -0800
Subject: [PATCH 456/860] xvideos: Fix for #30271

---
 youtube_dl/extractor/xvideos.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xvideos.py b/youtube_dl/extractor/xvideos.py
index 8fc64914c..e63d4690d 100644
--- a/youtube_dl/extractor/xvideos.py
+++ b/youtube_dl/extractor/xvideos.py
@@ -82,7 +82,7 @@ class XVideosIE(InfoExtractor):
         video_id = self._match_id(url)
 
         webpage = self._download_webpage(
-            'https://www.xvideos.com/video%s/' % video_id, video_id)
+            'https://www.xvideos.com/video%s/0' % video_id, video_id)
 
         mobj = re.search(r'<h1 class="inlineError">(.+?)</h1>', webpage)
         if mobj:

From 005339d6375f2d2a4cec962b1c1a157c1dffbf8f Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 8 Dec 2021 23:37:54 +0000
Subject: [PATCH 457/860] [applepodcasts] Support new AMP-ish page structure

---
 youtube_dl/extractor/applepodcasts.py | 43 ++++++++++++++++++++-------
 1 file changed, 33 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/applepodcasts.py b/youtube_dl/extractor/applepodcasts.py
index 6a74de758..f0186d4bf 100644
--- a/youtube_dl/extractor/applepodcasts.py
+++ b/youtube_dl/extractor/applepodcasts.py
@@ -3,7 +3,9 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
     clean_podcast_url,
+    get_element_by_class,
     int_or_none,
     parse_iso8601,
     try_get,
@@ -14,15 +16,15 @@ class ApplePodcastsIE(InfoExtractor):
     _VALID_URL = r'https?://podcasts\.apple\.com/(?:[^/]+/)?podcast(?:/[^/]+){1,2}.*?\bi=(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://podcasts.apple.com/us/podcast/207-whitney-webb-returns/id1135137367?i=1000482637777',
-        'md5': 'df02e6acb11c10e844946a39e7222b08',
+        'md5': '41dc31cd650143e530d9423b6b5a344f',
         'info_dict': {
             'id': '1000482637777',
             'ext': 'mp3',
             'title': '207 - Whitney Webb Returns',
-            'description': 'md5:13a73bade02d2e43737751e3987e1399',
+            'description': 'md5:75ef4316031df7b41ced4e7b987f79c6',
             'upload_date': '20200705',
-            'timestamp': 1593921600,
-            'duration': 6425,
+            'timestamp': 1593932400,
+            'duration': 6454,
             'series': 'The Tim Dillon Show',
         }
     }, {
@@ -39,17 +41,38 @@ class ApplePodcastsIE(InfoExtractor):
     def _real_extract(self, url):
         episode_id = self._match_id(url)
         webpage = self._download_webpage(url, episode_id)
-        ember_data = self._parse_json(self._search_regex(
-            r'id="shoebox-ember-data-store"[^>]*>\s*({.+?})\s*<',
-            webpage, 'ember data'), episode_id)
-        ember_data = ember_data.get(episode_id) or ember_data
-        episode = ember_data['data']['attributes']
+        episode_data = {}
+        ember_data = {}
+        # new page type 2021-11
+        amp_data = self._parse_json(self._search_regex(
+            r'(?s)id="shoebox-media-api-cache-amp-podcasts"[^>]*>\s*({.+?})\s*<',
+            webpage, 'AMP data', default='{}'), episode_id, fatal=False) or {}
+        amp_data = try_get(amp_data,
+                           lambda a: self._parse_json(
+                               next(a[x] for x in iter(a) if episode_id in x),
+                               episode_id),
+                           dict) or {}
+        amp_data = amp_data.get('d') or []
+        episode_data = try_get(
+            amp_data,
+            lambda a: next(x for x in a
+                           if x['type'] == 'podcast-episodes' and x['id'] == episode_id),
+            dict)
+        if not episode_data:
+            # try pre 2021-11 page type: TODO: consider deleting if no longer used
+            ember_data = self._parse_json(self._search_regex(
+                r'(?s)id="shoebox-ember-data-store"[^>]*>\s*({.+?})\s*<',
+                webpage, 'ember data'), episode_id) or {}
+            ember_data = ember_data.get(episode_id) or ember_data
+            episode_data = try_get(ember_data, lambda x: x['data'], dict)
+        episode = episode_data['attributes']
         description = episode.get('description') or {}
 
         series = None
-        for inc in (ember_data.get('included') or []):
+        for inc in (amp_data or ember_data.get('included') or []):
             if inc.get('type') == 'media/podcast':
                 series = try_get(inc, lambda x: x['attributes']['name'])
+        series = series or clean_html(get_element_by_class('podcast-header__identity', webpage))
 
         return {
             'id': episode_id,

From e00b0eab1e78ed822683b2689f60eab85514ac42 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 9 Dec 2021 00:55:04 +0000
Subject: [PATCH 458/860] [applepodcasts] Improve format extraction

Set acodec and vcodec, etc, to avoid breaking, eg, bestaudio
---
 youtube_dl/extractor/applepodcasts.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/applepodcasts.py b/youtube_dl/extractor/applepodcasts.py
index f0186d4bf..dd413a289 100644
--- a/youtube_dl/extractor/applepodcasts.py
+++ b/youtube_dl/extractor/applepodcasts.py
@@ -7,6 +7,7 @@ from ..utils import (
     clean_podcast_url,
     get_element_by_class,
     int_or_none,
+    parse_codecs,
     parse_iso8601,
     try_get,
 )
@@ -74,7 +75,7 @@ class ApplePodcastsIE(InfoExtractor):
                 series = try_get(inc, lambda x: x['attributes']['name'])
         series = series or clean_html(get_element_by_class('podcast-header__identity', webpage))
 
-        return {
+        info = [{
             'id': episode_id,
             'title': episode['name'],
             'url': clean_podcast_url(episode['assetUrl']),
@@ -82,4 +83,9 @@ class ApplePodcastsIE(InfoExtractor):
             'timestamp': parse_iso8601(episode.get('releaseDateTime')),
             'duration': int_or_none(episode.get('durationInMilliseconds'), 1000),
             'series': series,
-        }
+        }]
+        self._sort_formats(info)
+        info = info[0]
+        codecs = parse_codecs(info.get('ext', 'mp3'))
+        info.update(codecs)
+        return info

From 584715a803eef68f68fbbb8b72a022a699983197 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 9 Dec 2021 01:35:35 +0000
Subject: [PATCH 459/860] [applepodcasts] Extract default thumbnail image

---
 youtube_dl/extractor/applepodcasts.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/applepodcasts.py b/youtube_dl/extractor/applepodcasts.py
index dd413a289..95e0f663c 100644
--- a/youtube_dl/extractor/applepodcasts.py
+++ b/youtube_dl/extractor/applepodcasts.py
@@ -27,6 +27,7 @@ class ApplePodcastsIE(InfoExtractor):
             'timestamp': 1593932400,
             'duration': 6454,
             'series': 'The Tim Dillon Show',
+            'thumbnail': 're:.+[.](png|jpe?g|webp)',
         }
     }, {
         'url': 'https://podcasts.apple.com/podcast/207-whitney-webb-returns/id1135137367?i=1000482637777',
@@ -83,6 +84,7 @@ class ApplePodcastsIE(InfoExtractor):
             'timestamp': parse_iso8601(episode.get('releaseDateTime')),
             'duration': int_or_none(episode.get('durationInMilliseconds'), 1000),
             'series': series,
+            'thumbnail': self._og_search_thumbnail(webpage),
         }]
         self._sort_formats(info)
         info = info[0]

From 73e1ab6125eeea2b07942326cd2f1d6d9adff64e Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 6 Dec 2021 19:26:33 +0000
Subject: [PATCH 460/860] [test:download] Only extract enough videos for
 playlist_mincount

---
 test/parameters.json  | 1 -
 test/test_download.py | 1 +
 2 files changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/parameters.json b/test/parameters.json
index 65fd54428..864c9d130 100644
--- a/test/parameters.json
+++ b/test/parameters.json
@@ -18,7 +18,6 @@
     "noprogress": false, 
     "outtmpl": "%(id)s.%(ext)s", 
     "password": null, 
-    "playlistend": -1, 
     "playliststart": 1, 
     "prefer_free_formats": false, 
     "quiet": false, 
diff --git a/test/test_download.py b/test/test_download.py
index ebe820dfc..8e43cfa12 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -121,6 +121,7 @@ def generator(test_case, tname):
         params['outtmpl'] = tname + '_' + params['outtmpl']
         if is_playlist and 'playlist' not in test_case:
             params.setdefault('extract_flat', 'in_playlist')
+            params.setdefault('playlistend', test_case.get('playlist_mincount'))
             params.setdefault('skip_download', True)
 
         ydl = YoutubeDL(params, auto_init=False)

From 91278f4b6b5600e9ce65826ec9e7e38e7dba5937 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 6 Dec 2021 20:52:21 +0000
Subject: [PATCH 461/860] [niconico] Back-port extractor from yt-dlp

Add Nico search extractors, fix extraction
---
 youtube_dl/extractor/extractors.py |   9 +-
 youtube_dl/extractor/niconico.py   | 646 +++++++++++++++++++++--------
 2 files changed, 477 insertions(+), 178 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 4e9954c6a..e70daf2b1 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -789,7 +789,14 @@ from .nick import (
     NickNightIE,
     NickRuIE,
 )
-from .niconico import NiconicoIE, NiconicoPlaylistIE
+from .niconico import (
+    NiconicoIE,
+    NiconicoPlaylistIE,
+    NiconicoUserIE,
+    NicovideoSearchIE,
+    NicovideoSearchDateIE,
+    NicovideoSearchURLIE,
+)
 from .ninecninemedia import NineCNineMediaIE
 from .ninegag import NineGagIE
 from .ninenow import NineNowIE
diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index a85fc3d5c..756ad0e25 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -2,25 +2,28 @@
 from __future__ import unicode_literals
 
 import datetime
-import functools
+import itertools
 import json
-import math
+import re
 
-from .common import InfoExtractor
+from .common import InfoExtractor, SearchInfoExtractor
+from ..postprocessor.ffmpeg import FFmpegPostProcessor
 from ..compat import (
     compat_parse_qs,
+    compat_str,
     compat_urllib_parse_urlparse,
 )
 from ..utils import (
-    determine_ext,
-    dict_get,
     ExtractorError,
+    dict_get,
     float_or_none,
-    InAdvancePagedList,
     int_or_none,
+    OnDemandPagedList,
     parse_duration,
     parse_iso8601,
+    PostProcessingError,
     remove_start,
+    str_or_none,
     try_get,
     unified_timestamp,
     urlencode_postdata,
@@ -34,7 +37,7 @@ class NiconicoIE(InfoExtractor):
 
     _TESTS = [{
         'url': 'http://www.nicovideo.jp/watch/sm22312215',
-        'md5': 'd1a75c0823e2f629128c43e1212760f9',
+        'md5': 'a5bad06f1347452102953f323c69da34s',
         'info_dict': {
             'id': 'sm22312215',
             'ext': 'mp4',
@@ -162,6 +165,11 @@ class NiconicoIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.|secure\.|sp\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'
     _NETRC_MACHINE = 'niconico'
 
+    _API_HEADERS = {
+        'X-Frontend-ID': '6',
+        'X-Frontend-Version': '0'
+    }
+
     def _real_initialize(self):
         self._login()
 
@@ -191,37 +199,89 @@ class NiconicoIE(InfoExtractor):
             self._downloader.report_warning('unable to log in: bad username or password')
         return login_ok
 
-    def _extract_format_for_quality(self, api_data, video_id, audio_quality, video_quality):
-        def yesno(boolean):
-            return 'yes' if boolean else 'no'
+    def _get_heartbeat_info(self, info_dict):
 
-        session_api_data = api_data['video']['dmcInfo']['session_api']
-        session_api_endpoint = session_api_data['urls'][0]
+        video_id, video_src_id, audio_src_id = info_dict['url'].split(':')[1].split('/')
 
-        format_id = '-'.join(map(lambda s: remove_start(s['id'], 'archive_'), [video_quality, audio_quality]))
+        api_data = (
+            info_dict.get('_api_data')
+            or self._parse_json(
+                self._html_search_regex(
+                    'data-api-data="([^"]+)"',
+                    self._download_webpage('http://www.nicovideo.jp/watch/' + video_id, video_id),
+                    'API data', default='{}'),
+                video_id))
+
+        session_api_data = try_get(api_data, lambda x: x['media']['delivery']['movie']['session'])
+        session_api_endpoint = try_get(session_api_data, lambda x: x['urls'][0])
+
+        def ping():
+            status = try_get(
+                self._download_json(
+                    'https://nvapi.nicovideo.jp/v1/2ab0cbaa/watch', video_id,
+                    query={'t': try_get(api_data, lambda x: x['media']['delivery']['trackingId'])},
+                    note='Acquiring permission for downloading video',
+                    headers=self._API_HEADERS),
+                lambda x: x['meta']['status'])
+            if status != 200:
+                self.report_warning('Failed to acquire permission for playing video. The video may not download.')
+
+        yesno = lambda x: 'yes' if x else 'no'
+
+        # m3u8 (encryption)
+        if try_get(api_data, lambda x: x['media']['delivery']['encryption']) is not None:
+            protocol = 'm3u8'
+            encryption = self._parse_json(session_api_data['token'], video_id)['hls_encryption']
+            session_api_http_parameters = {
+                'parameters': {
+                    'hls_parameters': {
+                        'encryption': {
+                            encryption: {
+                                'encrypted_key': try_get(api_data, lambda x: x['media']['delivery']['encryption']['encryptedKey']),
+                                'key_uri': try_get(api_data, lambda x: x['media']['delivery']['encryption']['keyUri'])
+                            }
+                        },
+                        'transfer_preset': '',
+                        'use_ssl': yesno(session_api_endpoint['isSsl']),
+                        'use_well_known_port': yesno(session_api_endpoint['isWellKnownPort']),
+                        'segment_duration': 6000,
+                    }
+                }
+            }
+        # http
+        else:
+            protocol = 'http'
+            session_api_http_parameters = {
+                'parameters': {
+                    'http_output_download_parameters': {
+                        'use_ssl': yesno(session_api_endpoint['isSsl']),
+                        'use_well_known_port': yesno(session_api_endpoint['isWellKnownPort']),
+                    }
+                }
+            }
 
         session_response = self._download_json(
             session_api_endpoint['url'], video_id,
             query={'_format': 'json'},
             headers={'Content-Type': 'application/json'},
-            note='Downloading JSON metadata for %s' % format_id,
+            note='Downloading JSON metadata for %s' % info_dict['format_id'],
             data=json.dumps({
                 'session': {
                     'client_info': {
-                        'player_id': session_api_data['player_id'],
+                        'player_id': session_api_data.get('playerId'),
                     },
                     'content_auth': {
-                        'auth_type': session_api_data['auth_types'][session_api_data['protocols'][0]],
-                        'content_key_timeout': session_api_data['content_key_timeout'],
+                        'auth_type': try_get(session_api_data, lambda x: x['authTypes'][session_api_data['protocols'][0]]),
+                        'content_key_timeout': session_api_data.get('contentKeyTimeout'),
                         'service_id': 'nicovideo',
-                        'service_user_id': session_api_data['service_user_id']
+                        'service_user_id': session_api_data.get('serviceUserId')
                     },
-                    'content_id': session_api_data['content_id'],
+                    'content_id': session_api_data.get('contentId'),
                     'content_src_id_sets': [{
                         'content_src_ids': [{
                             'src_id_to_mux': {
-                                'audio_src_ids': [audio_quality['id']],
-                                'video_src_ids': [video_quality['id']],
+                                'audio_src_ids': [audio_src_id],
+                                'video_src_ids': [video_src_id],
                             }
                         }]
                     }],
@@ -229,52 +289,81 @@ class NiconicoIE(InfoExtractor):
                     'content_uri': '',
                     'keep_method': {
                         'heartbeat': {
-                            'lifetime': session_api_data['heartbeat_lifetime']
+                            'lifetime': session_api_data.get('heartbeatLifetime')
                         }
                     },
-                    'priority': session_api_data['priority'],
+                    'priority': session_api_data.get('priority'),
                     'protocol': {
                         'name': 'http',
                         'parameters': {
-                            'http_parameters': {
-                                'parameters': {
-                                    'http_output_download_parameters': {
-                                        'use_ssl': yesno(session_api_endpoint['is_ssl']),
-                                        'use_well_known_port': yesno(session_api_endpoint['is_well_known_port']),
-                                    }
-                                }
-                            }
+                            'http_parameters': session_api_http_parameters
                         }
                     },
-                    'recipe_id': session_api_data['recipe_id'],
+                    'recipe_id': session_api_data.get('recipeId'),
                     'session_operation_auth': {
                         'session_operation_auth_by_signature': {
-                            'signature': session_api_data['signature'],
-                            'token': session_api_data['token'],
+                            'signature': session_api_data.get('signature'),
+                            'token': session_api_data.get('token'),
                         }
                     },
                     'timing_constraint': 'unlimited'
                 }
             }).encode())
 
-        resolution = video_quality.get('resolution', {})
+        info_dict['url'] = session_response['data']['session']['content_uri']
+        info_dict['protocol'] = protocol
+
+        # get heartbeat info
+        heartbeat_info_dict = {
+            'url': session_api_endpoint['url'] + '/' + session_response['data']['session']['id'] + '?_format=json&_method=PUT',
+            'data': json.dumps(session_response['data']),
+            # interval, convert milliseconds to seconds, then halve to make a buffer.
+            'interval': float_or_none(session_api_data.get('heartbeatLifetime'), scale=3000),
+            'ping': ping
+        }
+
+        return info_dict, heartbeat_info_dict
+
+    def _extract_format_for_quality(self, api_data, video_id, audio_quality, video_quality):
+        def parse_format_id(id_code):
+            mobj = re.match(r'''(?x)
+                    (?:archive_)?
+                    (?:(?P<codec>[^_]+)_)?
+                    (?:(?P<br>[\d]+)kbps_)?
+                    (?:(?P<res>[\d+]+)p_)?
+                ''', '%s_' % id_code)
+            return mobj.groupdict() if mobj else {}
+
+        protocol = 'niconico_dmc'
+        format_id = '-'.join(map(lambda s: remove_start(s['id'], 'archive_'), [video_quality, audio_quality]))
+        vdict = parse_format_id(video_quality['id'])
+        adict = parse_format_id(audio_quality['id'])
+        resolution = try_get(video_quality, lambda x: x['metadata']['resolution'], dict) or {'height': vdict.get('res')}
+        vbr = try_get(video_quality, lambda x: x['metadata']['bitrate'], float)
 
         return {
-            'url': session_response['data']['session']['content_uri'],
+            'url': '%s:%s/%s/%s' % (protocol, video_id, video_quality['id'], audio_quality['id']),
             'format_id': format_id,
+            'format_note': 'DMC %s' % try_get(video_quality, lambda x: x['metadata']['label'], compat_str),
             'ext': 'mp4',  # Session API are used in HTML5, which always serves mp4
-            'abr': float_or_none(audio_quality.get('bitrate'), 1000),
-            'vbr': float_or_none(video_quality.get('bitrate'), 1000),
-            'height': resolution.get('height'),
-            'width': resolution.get('width'),
+            'vcodec': vdict.get('codec'),
+            'acodec': adict.get('codec'),
+            'vbr': float_or_none(vbr, 1000) or float_or_none(vdict.get('br')),
+            'abr': float_or_none(audio_quality.get('bitrate'), 1000) or float_or_none(adict.get('br')),
+            'height': int_or_none(resolution.get('height', vdict.get('res'))),
+            'width': int_or_none(resolution.get('width')),
+            'quality': -2 if 'low' in format_id else -1,  # Default quality value is -1
+            'protocol': protocol,
+            'http_headers': {
+                'Origin': 'https://www.nicovideo.jp',
+                'Referer': 'https://www.nicovideo.jp/watch/' + video_id,
+            }
         }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        # Get video webpage. We are not actually interested in it for normal
-        # cases, but need the cookies in order to be able to download the
-        # info webpage
+        # Get video webpage for API data.
         webpage, handle = self._download_webpage_handle(
             'http://www.nicovideo.jp/watch/' + video_id, video_id)
         if video_id.startswith('so'):
@@ -284,86 +373,136 @@ class NiconicoIE(InfoExtractor):
             'data-api-data="([^"]+)"', webpage,
             'API data', default='{}'), video_id)
 
-        def _format_id_from_url(video_url):
-            return 'economy' if video_real_url.endswith('low') else 'normal'
+        def get_video_info_web(items):
+            return dict_get(api_data['video'], items)
 
-        try:
-            video_real_url = api_data['video']['smileInfo']['url']
-        except KeyError:  # Flash videos
-            # Get flv info
-            flv_info_webpage = self._download_webpage(
-                'http://flapi.nicovideo.jp/api/getflv/' + video_id + '?as3=1',
-                video_id, 'Downloading flv info')
+        # Get video info
+        video_info_xml = self._download_xml(
+            'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id,
+            video_id, note='Downloading video info page')
 
-            flv_info = compat_parse_qs(flv_info_webpage)
-            if 'url' not in flv_info:
-                if 'deleted' in flv_info:
-                    raise ExtractorError('The video has been deleted.',
-                                         expected=True)
-                elif 'closed' in flv_info:
-                    raise ExtractorError('Niconico videos now require logging in',
-                                         expected=True)
-                elif 'error' in flv_info:
-                    raise ExtractorError('%s reports error: %s' % (
-                        self.IE_NAME, flv_info['error'][0]), expected=True)
-                else:
-                    raise ExtractorError('Unable to find video URL')
+        def get_video_info_xml(items):
+            if not isinstance(items, list):
+                items = [items]
+            for item in items:
+                ret = xpath_text(video_info_xml, './/' + item)
+                if ret:
+                    return ret
 
-            video_info_xml = self._download_xml(
-                'http://ext.nicovideo.jp/api/getthumbinfo/' + video_id,
-                video_id, note='Downloading video info page')
+        if get_video_info_xml('error'):
+            error_code = get_video_info_xml('code')
 
-            def get_video_info(items):
-                if not isinstance(items, list):
-                    items = [items]
-                for item in items:
-                    ret = xpath_text(video_info_xml, './/' + item)
-                    if ret:
-                        return ret
+            if error_code == 'DELETED':
+                raise ExtractorError('The video has been deleted.',
+                                     expected=True)
+            elif error_code == 'NOT_FOUND':
+                raise ExtractorError('The video is not found.',
+                                     expected=True)
+            elif error_code == 'COMMUNITY':
+                self.to_screen('%s: The video is community members only.' % video_id)
+            else:
+                raise ExtractorError('%s reports error: %s' % (self.IE_NAME, error_code))
 
-            video_real_url = flv_info['url'][0]
+        # Start extracting video formats
+        formats = []
 
-            extension = get_video_info('movie_type')
-            if not extension:
-                extension = determine_ext(video_real_url)
+        # Get HTML5 videos info
+        quality_info = try_get(api_data, lambda x: x['media']['delivery']['movie'])
+        if not quality_info:
+            raise ExtractorError('The video can\'t be downloaded', expected=True)
 
-            formats = [{
-                'url': video_real_url,
-                'ext': extension,
-                'format_id': _format_id_from_url(video_real_url),
-            }]
-        else:
-            formats = []
+        for audio_quality in quality_info.get('audios') or {}:
+            for video_quality in quality_info.get('videos') or {}:
+                if not audio_quality.get('isAvailable') or not video_quality.get('isAvailable'):
+                    continue
+                formats.append(self._extract_format_for_quality(
+                    api_data, video_id, audio_quality, video_quality))
 
-            dmc_info = api_data['video'].get('dmcInfo')
-            if dmc_info:  # "New" HTML5 videos
-                quality_info = dmc_info['quality']
-                for audio_quality in quality_info['audios']:
-                    for video_quality in quality_info['videos']:
-                        if not audio_quality['available'] or not video_quality['available']:
-                            continue
-                        formats.append(self._extract_format_for_quality(
-                            api_data, video_id, audio_quality, video_quality))
+        # Get flv/swf info
+        timestamp = None
+        video_real_url = try_get(api_data, lambda x: x['video']['smileInfo']['url'])
+        if video_real_url:
+            is_economy = video_real_url.endswith('low')
 
-                self._sort_formats(formats)
-            else:  # "Old" HTML5 videos
-                formats = [{
+            if is_economy:
+                self.report_warning('Site is currently in economy mode! You will only have access to lower quality streams')
+
+            # Invoking ffprobe to determine resolution
+            pp = FFmpegPostProcessor(self._downloader)
+            cookies = self._get_cookies('https://nicovideo.jp').output(header='', sep='; path=/; domain=nicovideo.jp;\n')
+
+            self.to_screen('%s: %s' % (video_id, 'Checking smile format with ffprobe'))
+
+            try:
+                metadata = pp.get_metadata_object(video_real_url, ['-cookies', cookies])
+            except PostProcessingError as err:
+                raise ExtractorError(err.msg, expected=True)
+
+            v_stream = a_stream = {}
+
+            # Some complex swf files doesn't have video stream (e.g. nm4809023)
+            for stream in metadata['streams']:
+                if stream['codec_type'] == 'video':
+                    v_stream = stream
+                elif stream['codec_type'] == 'audio':
+                    a_stream = stream
+
+            # Community restricted videos seem to have issues with the thumb API not returning anything at all
+            filesize = int(
+                (get_video_info_xml('size_high') if not is_economy else get_video_info_xml('size_low'))
+                or metadata['format']['size']
+            )
+            extension = (
+                get_video_info_xml('movie_type')
+                or 'mp4' if 'mp4' in metadata['format']['format_name'] else metadata['format']['format_name']
+            )
+
+            # 'creation_time' tag on video stream of re-encoded SMILEVIDEO mp4 files are '1970-01-01T00:00:00.000000Z'.
+            timestamp = (
+                parse_iso8601(get_video_info_web('first_retrieve'))
+                or unified_timestamp(get_video_info_web('postedDateTime'))
+            )
+            metadata_timestamp = (
+                parse_iso8601(try_get(v_stream, lambda x: x['tags']['creation_time']))
+                or timestamp if extension != 'mp4' else 0
+            )
+
+            # According to compconf, smile videos from pre-2017 are always better quality than their DMC counterparts
+            smile_threshold_timestamp = parse_iso8601('2016-12-08T00:00:00+09:00')
+
+            is_source = timestamp < smile_threshold_timestamp or metadata_timestamp > 0
+
+            # If movie file size is unstable, old server movie is not source movie.
+            if filesize > 1:
+                formats.append({
                     'url': video_real_url,
-                    'ext': 'mp4',
-                    'format_id': _format_id_from_url(video_real_url),
-                }]
+                    'format_id': 'smile' if not is_economy else 'smile_low',
+                    'format_note': 'SMILEVIDEO source' if not is_economy else 'SMILEVIDEO low quality',
+                    'ext': extension,
+                    'container': extension,
+                    'vcodec': v_stream.get('codec_name'),
+                    'acodec': a_stream.get('codec_name'),
+                    # Some complex swf files doesn't have total bit rate metadata (e.g. nm6049209)
+                    'tbr': int_or_none(metadata['format'].get('bit_rate'), scale=1000),
+                    'vbr': int_or_none(v_stream.get('bit_rate'), scale=1000),
+                    'abr': int_or_none(a_stream.get('bit_rate'), scale=1000),
+                    'height': int_or_none(v_stream.get('height')),
+                    'width': int_or_none(v_stream.get('width')),
+                    'source_preference': 5 if not is_economy else -2,
+                    'quality': 5 if is_source and not is_economy else None,
+                    'filesize': filesize
+                })
 
-            def get_video_info(items):
-                return dict_get(api_data['video'], items)
+        self._sort_formats(formats)
 
         # Start extracting information
-        title = get_video_info('title')
-        if not title:
-            title = self._og_search_title(webpage, default=None)
-        if not title:
-            title = self._html_search_regex(
+        title = (
+            get_video_info_xml('title')  # prefer to get the untranslated original title
+            or get_video_info_web(['originalTitle', 'title'])
+            or self._og_search_title(webpage, default=None)
+            or self._html_search_regex(
                 r'<span[^>]+class="videoHeaderTitle"[^>]*>([^<]+)</span>',
-                webpage, 'video title')
+                webpage, 'video title'))
 
         watch_api_data_string = self._html_search_regex(
             r'<div[^>]+id="watchAPIDataContainer"[^>]+>([^<]+)</div>',
@@ -372,14 +511,15 @@ class NiconicoIE(InfoExtractor):
         video_detail = watch_api_data.get('videoDetail', {})
 
         thumbnail = (
-            get_video_info(['thumbnail_url', 'thumbnailURL'])
+            self._html_search_regex(r'<meta property="og:image" content="([^"]+)">', webpage, 'thumbnail data', default=None)
+            or dict_get(  # choose highest from 720p to 240p
+                get_video_info_web('thumbnail'),
+                ['ogp', 'player', 'largeUrl', 'middleUrl', 'url'])
             or self._html_search_meta('image', webpage, 'thumbnail', default=None)
             or video_detail.get('thumbnail'))
 
-        description = get_video_info('description')
+        description = get_video_info_web('description')
 
-        timestamp = (parse_iso8601(get_video_info('first_retrieve'))
-                     or unified_timestamp(get_video_info('postedDateTime')))
         if not timestamp:
             match = self._html_search_meta('datePublished', webpage, 'date published', default=None)
             if match:
@@ -388,19 +528,25 @@ class NiconicoIE(InfoExtractor):
             timestamp = parse_iso8601(
                 video_detail['postedAt'].replace('/', '-'),
                 delimiter=' ', timezone=datetime.timedelta(hours=9))
+        timestamp = timestamp or try_get(api_data, lambda x: parse_iso8601(x['video']['registeredAt']))
 
-        view_count = int_or_none(get_video_info(['view_counter', 'viewCount']))
+        view_count = int_or_none(get_video_info_web(['view_counter', 'viewCount']))
         if not view_count:
             match = self._html_search_regex(
                 r'>Views: <strong[^>]*>([^<]+)</strong>',
                 webpage, 'view count', default=None)
             if match:
                 view_count = int_or_none(match.replace(',', ''))
-        view_count = view_count or video_detail.get('viewCount')
+        view_count = (
+            view_count
+            or video_detail.get('viewCount')
+            or try_get(api_data, lambda x: x['video']['count']['view']))
+
+        comment_count = (
+            int_or_none(get_video_info_web('comment_num'))
+            or video_detail.get('commentCount')
+            or try_get(api_data, lambda x: x['video']['count']['comment']))
 
-        comment_count = (int_or_none(get_video_info('comment_num'))
-                         or video_detail.get('commentCount')
-                         or try_get(api_data, lambda x: x['thread']['commentCount']))
         if not comment_count:
             match = self._html_search_regex(
                 r'>Comments: <strong[^>]*>([^<]+)</strong>',
@@ -409,22 +555,41 @@ class NiconicoIE(InfoExtractor):
                 comment_count = int_or_none(match.replace(',', ''))
 
         duration = (parse_duration(
-            get_video_info('length')
+            get_video_info_web('length')
             or self._html_search_meta(
                 'video:duration', webpage, 'video duration', default=None))
             or video_detail.get('length')
-            or get_video_info('duration'))
+            or get_video_info_web('duration'))
 
-        webpage_url = get_video_info('watch_url') or url
+        webpage_url = get_video_info_web('watch_url') or url
+
+        # for channel movie and community movie
+        channel_id = try_get(
+            api_data,
+            (lambda x: x['channel']['globalId'],
+             lambda x: x['community']['globalId']))
+        channel = try_get(
+            api_data,
+            (lambda x: x['channel']['name'],
+             lambda x: x['community']['name']))
 
         # Note: cannot use api_data.get('owner', {}) because owner may be set to "null"
         # in the JSON, which will cause None to be returned instead of {}.
         owner = try_get(api_data, lambda x: x.get('owner'), dict) or {}
-        uploader_id = get_video_info(['ch_id', 'user_id']) or owner.get('id')
-        uploader = get_video_info(['ch_name', 'user_nickname']) or owner.get('nickname')
+        uploader_id = str_or_none(
+            get_video_info_web(['ch_id', 'user_id'])
+            or owner.get('id')
+            or channel_id
+        )
+        uploader = (
+            get_video_info_web(['ch_name', 'user_nickname'])
+            or owner.get('nickname')
+            or channel
+        )
 
         return {
             'id': video_id,
+            '_api_data': api_data,
             'title': title,
             'formats': formats,
             'thumbnail': thumbnail,
@@ -432,6 +597,8 @@ class NiconicoIE(InfoExtractor):
             'uploader': uploader,
             'timestamp': timestamp,
             'uploader_id': uploader_id,
+            'channel': channel,
+            'channel_id': channel_id,
             'view_count': view_count,
             'comment_count': comment_count,
             'duration': duration,
@@ -440,7 +607,7 @@ class NiconicoIE(InfoExtractor):
 
 
 class NiconicoPlaylistIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?nicovideo\.jp/(?:user/\d+/)?mylist/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?nicovideo\.jp/(?:user/\d+/|my/)?mylist/(?P<id>\d+)'
 
     _TESTS = [{
         'url': 'http://www.nicovideo.jp/mylist/27411728',
@@ -456,60 +623,185 @@ class NiconicoPlaylistIE(InfoExtractor):
         'url': 'https://www.nicovideo.jp/user/805442/mylist/27411728',
         'only_matching': True,
     }]
-    _PAGE_SIZE = 100
 
-    def _call_api(self, list_id, resource, query):
-        return self._download_json(
-            'https://nvapi.nicovideo.jp/v2/mylists/' + list_id, list_id,
-            'Downloading %s JSON metatdata' % resource, query=query,
-            headers={'X-Frontend-Id': 6})['data']['mylist']
-
-    def _parse_owner(self, item):
-        owner = item.get('owner') or {}
-        if owner:
-            return {
-                'uploader': owner.get('name'),
-                'uploader_id': owner.get('id'),
-            }
-        return {}
-
-    def _fetch_page(self, list_id, page):
-        page += 1
-        items = self._call_api(list_id, 'page %d' % page, {
-            'page': page,
-            'pageSize': self._PAGE_SIZE,
-        })['items']
-        for item in items:
-            video = item.get('video') or {}
-            video_id = video.get('id')
-            if not video_id:
-                continue
-            count = video.get('count') or {}
-            get_count = lambda x: int_or_none(count.get(x))
-            info = {
-                '_type': 'url',
-                'id': video_id,
-                'title': video.get('title'),
-                'url': 'https://www.nicovideo.jp/watch/' + video_id,
-                'description': video.get('shortDescription'),
-                'duration': int_or_none(video.get('duration')),
-                'view_count': get_count('view'),
-                'comment_count': get_count('comment'),
-                'ie_key': NiconicoIE.ie_key(),
-            }
-            info.update(self._parse_owner(video))
-            yield info
+    _API_HEADERS = {
+        'X-Frontend-ID': '6',
+        'X-Frontend-Version': '0'
+    }
 
     def _real_extract(self, url):
         list_id = self._match_id(url)
-        mylist = self._call_api(list_id, 'list', {
-            'pageSize': 1,
-        })
-        entries = InAdvancePagedList(
-            functools.partial(self._fetch_page, list_id),
-            math.ceil(mylist['totalItemCount'] / self._PAGE_SIZE),
-            self._PAGE_SIZE)
-        result = self.playlist_result(
-            entries, list_id, mylist.get('name'), mylist.get('description'))
-        result.update(self._parse_owner(mylist))
-        return result
+
+        def get_page_data(pagenum, pagesize):
+            return self._download_json(
+                'http://nvapi.nicovideo.jp/v2/mylists/' + list_id, list_id,
+                query={'page': 1 + pagenum, 'pageSize': pagesize},
+                headers=self._API_HEADERS).get('data').get('mylist')
+
+        data = get_page_data(0, 1)
+        title = data.get('name')
+        description = data.get('description')
+        uploader = data.get('owner').get('name')
+        uploader_id = data.get('owner').get('id')
+
+        def pagefunc(pagenum):
+            data = get_page_data(pagenum, 25)
+            return ({
+                '_type': 'url',
+                'url': 'http://www.nicovideo.jp/watch/' + item.get('watchId'),
+            } for item in data.get('items'))
+
+        return {
+            '_type': 'playlist',
+            'id': list_id,
+            'title': title,
+            'description': description,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+            'entries': OnDemandPagedList(pagefunc, 25),
+        }
+
+
+class NicovideoSearchBaseIE(InfoExtractor):
+    _MAX_RESULTS = float('inf')
+
+    def _entries(self, url, item_id, query=None, note='Downloading page %(page)s'):
+        query = query or {}
+        pages = [query['page']] if 'page' in query else itertools.count(1)
+        for page_num in pages:
+            query['page'] = str(page_num)
+            webpage = self._download_webpage(url, item_id, query=query, note=note % {'page': page_num})
+            results = re.findall(r'(?<=data-video-id=)["\']?(?P<videoid>.+?)(?=["\'])', webpage)
+            for item in results:
+                yield self.url_result('http://www.nicovideo.jp/watch/%s' % item, 'Niconico', item)
+            if not results:
+                break
+
+    def _get_n_results(self, query, n):
+        entries = self._entries(self._proto_relative_url('//www.nicovideo.jp/search/%s' % query), query)
+        if n < self._MAX_RESULTS:
+            entries = itertools.islice(entries, 0, n)
+        return self.playlist_result(entries, query, query)
+
+
+class NicovideoSearchIE(NicovideoSearchBaseIE, SearchInfoExtractor):
+    IE_DESC = 'Nico video search'
+    IE_NAME = 'nicovideo:search'
+    _SEARCH_KEY = 'nicosearch'
+
+    def _search_results(self, query):
+        return self._entries(
+            self._proto_relative_url('//www.nicovideo.jp/search/%s' % query), query)
+
+
+class NicovideoSearchURLIE(NicovideoSearchBaseIE):
+    IE_NAME = '%s_url' % NicovideoSearchIE.IE_NAME
+    IE_DESC = 'Nico video search URLs'
+    _VALID_URL = r'https?://(?:www\.)?nicovideo\.jp/search/(?P<id>[^?#&]+)?'
+    _TESTS = [{
+        'url': 'http://www.nicovideo.jp/search/sm9',
+        'info_dict': {
+            'id': 'sm9',
+            'title': 'sm9'
+        },
+        'playlist_mincount': 40,
+    }, {
+        'url': 'https://www.nicovideo.jp/search/sm9?sort=h&order=d&end=2020-12-31&start=2020-01-01',
+        'info_dict': {
+            'id': 'sm9',
+            'title': 'sm9'
+        },
+        'playlist_count': 31,
+    }]
+
+    def _real_extract(self, url):
+        query = self._match_id(url)
+        return self.playlist_result(self._entries(url, query), query, query)
+
+
+class NicovideoSearchDateIE(NicovideoSearchBaseIE, SearchInfoExtractor):
+    IE_DESC = 'Nico video search, newest first'
+    IE_NAME = '%s:date' % NicovideoSearchIE.IE_NAME
+    _SEARCH_KEY = 'nicosearchdate'
+
+    _TESTS = [{
+        'url': 'nicosearchdateall:a',
+        'info_dict': {
+            'id': 'a',
+            'title': 'a'
+        },
+        'playlist_mincount': 1610,
+    }]
+
+    _START_DATE = datetime.date(2007, 1, 1)
+    _RESULTS_PER_PAGE = 32
+    _MAX_PAGES = 50
+
+    def _entries(self, url, item_id, start_date=None, end_date=None):
+        start_date, end_date = start_date or self._START_DATE, end_date or datetime.datetime.now().date()
+
+        # If the last page has a full page of videos, we need to break down the query interval further
+        last_page_len = len(list(self._get_entries_for_date(
+            url, item_id, start_date, end_date, self._MAX_PAGES,
+            note='Checking number of videos from {0} to {1}'.format(start_date, end_date))))
+        if (last_page_len == self._RESULTS_PER_PAGE and start_date != end_date):
+            midpoint = start_date + ((end_date - start_date) // 2)
+            for entry in itertools.chain(
+                    iter(self._entries(url, item_id, midpoint, end_date)),
+                    iter(self._entries(url, item_id, start_date, midpoint))):
+                yield entry
+        else:
+            self.to_screen('{0}: Downloading results from {1} to {2}'.format(item_id, start_date, end_date))
+            for entry in iter(self._get_entries_for_date(
+                    url, item_id, start_date, end_date, note='    Downloading page %(page)s')):
+                yield entry
+
+    def _get_entries_for_date(self, url, item_id, start_date, end_date=None, page_num=None, note=None):
+        query = {
+            'start': compat_str(start_date),
+            'end': compat_str(end_date or start_date),
+            'sort': 'f',
+            'order': 'd',
+        }
+        if page_num:
+            query['page'] = compat_str(page_num)
+
+        for entry in iter(super(NicovideoSearchDateIE, self)._entries(url, item_id, query=query, note=note)):
+            yield entry
+
+
+class NiconicoUserIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?nicovideo\.jp/user/(?P<id>\d+)/?(?:$|[#?])'
+    _TEST = {
+        'url': 'https://www.nicovideo.jp/user/419948',
+        'info_dict': {
+            'id': '419948',
+        },
+        'playlist_mincount': 101,
+    }
+    _API_URL = "https://nvapi.nicovideo.jp/v1/users/%s/videos?sortKey=registeredAt&sortOrder=desc&pageSize=%s&page=%s"
+    _PAGE_SIZE = 100
+
+    _API_HEADERS = {
+        'X-Frontend-ID': '6',
+        'X-Frontend-Version': '0'
+    }
+
+    def _entries(self, list_id):
+        total_count = 1
+        count = page_num = 0
+        while count < total_count:
+            json_parsed = self._download_json(
+                self._API_URL % (list_id, self._PAGE_SIZE, page_num + 1), list_id,
+                headers=self._API_HEADERS,
+                note='Downloading JSON metadata%s' % (' page %d' % page_num if page_num else ''))
+            if not page_num:
+                total_count = int_or_none(json_parsed['data'].get('totalCount'))
+            for entry in json_parsed["data"]["items"]:
+                count += 1
+                yield self.url_result('https://www.nicovideo.jp/watch/%s' % entry['id'])
+            page_num += 1
+
+    def _real_extract(self, url):
+        list_id = self._match_id(url)
+        return self.playlist_result(self._entries(list_id), list_id)

From 92d73ef3936ed6de9770f613fddf2260731becc9 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 7 Dec 2021 23:30:30 +0000
Subject: [PATCH 462/860] [niconico] Implement heartbeat for download

---
 youtube_dl/downloader/__init__.py | 25 ++++++++----
 youtube_dl/downloader/niconico.py | 66 +++++++++++++++++++++++++++++++
 youtube_dl/extractor/niconico.py  | 18 +++++++++
 3 files changed, 101 insertions(+), 8 deletions(-)
 create mode 100644 youtube_dl/downloader/niconico.py

diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index 2e485df9d..d8f2fa342 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -1,22 +1,31 @@
 from __future__ import unicode_literals
 
+from ..utils import (
+    determine_protocol,
+)
+
+
+def get_suitable_downloader(info_dict, params={}):
+    info_dict['protocol'] = determine_protocol(info_dict)
+    info_copy = info_dict.copy()
+    return _get_suitable_downloader(info_copy, params)
+
+
+# Some of these require get_suitable_downloader
 from .common import FileDownloader
+from .dash import DashSegmentsFD
 from .f4m import F4mFD
 from .hls import HlsFD
 from .http import HttpFD
 from .rtmp import RtmpFD
-from .dash import DashSegmentsFD
 from .rtsp import RtspFD
 from .ism import IsmFD
+from .niconico import NiconicoDmcFD
 from .external import (
     get_external_downloader,
     FFmpegFD,
 )
 
-from ..utils import (
-    determine_protocol,
-)
-
 PROTOCOL_MAP = {
     'rtmp': RtmpFD,
     'm3u8_native': HlsFD,
@@ -26,13 +35,12 @@ PROTOCOL_MAP = {
     'f4m': F4mFD,
     'http_dash_segments': DashSegmentsFD,
     'ism': IsmFD,
+    'niconico_dmc': NiconicoDmcFD,
 }
 
 
-def get_suitable_downloader(info_dict, params={}):
+def _get_suitable_downloader(info_dict, params={}):
     """Get the downloader class that can handle the info dict."""
-    protocol = determine_protocol(info_dict)
-    info_dict['protocol'] = protocol
 
     # if (info_dict.get('start_time') or info_dict.get('end_time')) and not info_dict.get('requested_formats') and FFmpegFD.can_download(info_dict):
     #     return FFmpegFD
@@ -43,6 +51,7 @@ def get_suitable_downloader(info_dict, params={}):
         if ed.can_download(info_dict):
             return ed
 
+    protocol = info_dict['protocol']
     if protocol.startswith('m3u8') and info_dict.get('is_live'):
         return FFmpegFD
 
diff --git a/youtube_dl/downloader/niconico.py b/youtube_dl/downloader/niconico.py
new file mode 100644
index 000000000..6392c9989
--- /dev/null
+++ b/youtube_dl/downloader/niconico.py
@@ -0,0 +1,66 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+try:
+    import threading
+except ImportError:
+    threading = None
+
+from .common import FileDownloader
+from ..downloader import get_suitable_downloader
+from ..extractor.niconico import NiconicoIE
+from ..utils import sanitized_Request
+
+
+class NiconicoDmcFD(FileDownloader):
+    """ Downloading niconico douga from DMC with heartbeat """
+
+    FD_NAME = 'niconico_dmc'
+
+    def real_download(self, filename, info_dict):
+        self.to_screen('[%s] Downloading from DMC' % self.FD_NAME)
+
+        ie = NiconicoIE(self.ydl)
+        info_dict, heartbeat_info_dict = ie._get_heartbeat_info(info_dict)
+
+        fd = get_suitable_downloader(info_dict, params=self.params)(self.ydl, self.params)
+        for ph in self._progress_hooks:
+            fd.add_progress_hook(ph)
+
+        if not threading:
+            self.to_screen('[%s] Threading for Heartbeat not available' % self.FD_NAME)
+            return fd.real_download(filename, info_dict)
+
+        success = download_complete = False
+        timer = [None]
+        heartbeat_lock = threading.Lock()
+        heartbeat_url = heartbeat_info_dict['url']
+        heartbeat_data = heartbeat_info_dict['data'].encode()
+        heartbeat_interval = heartbeat_info_dict.get('interval', 30)
+
+        request = sanitized_Request(heartbeat_url, heartbeat_data)
+
+        def heartbeat():
+            try:
+                self.ydl.urlopen(request).read()
+            except Exception:
+                self.to_screen('[%s] Heartbeat failed' % self.FD_NAME)
+
+            with heartbeat_lock:
+                if not download_complete:
+                    timer[0] = threading.Timer(heartbeat_interval, heartbeat)
+                    timer[0].start()
+
+        heartbeat_info_dict['ping']()
+        self.to_screen('[%s] Heartbeat with %d second interval ...' % (self.FD_NAME, heartbeat_interval))
+        try:
+            heartbeat()
+            if type(fd).__name__ == 'HlsFD':
+                info_dict.update(ie._extract_m3u8_formats(info_dict['url'], info_dict['id'])[0])
+            success = fd.real_download(filename, info_dict)
+        finally:
+            if heartbeat_lock:
+                with heartbeat_lock:
+                    timer[0].cancel()
+                    download_complete = True
+            return success
diff --git a/youtube_dl/extractor/niconico.py b/youtube_dl/extractor/niconico.py
index 756ad0e25..93f813968 100644
--- a/youtube_dl/extractor/niconico.py
+++ b/youtube_dl/extractor/niconico.py
@@ -160,6 +160,24 @@ class NiconicoIE(InfoExtractor):
     }, {
         'url': 'http://sp.nicovideo.jp/watch/sm28964488?ss_pos=1&cp_in=wt_tg',
         'only_matching': True,
+    }, {
+        # DMC video with heartbeat
+        'url': 'https://www.nicovideo.jp/watch/sm34815188',
+        'md5': '9360c6e1f1519d7759e2fe8e1326ae83',
+        'info_dict': {
+            'id': 'sm34815188',
+            'ext': 'mp4',
+            'title': 'md5:aee93e9f3366db72f902f6cd5d389cb7',
+            'description': 'md5:7b9149fc7a00ab053cafaf5c19662704',
+            'thumbnail': r're:https?://.*',
+            'uploader': 'md5:2762e18fa74dbb40aa1ad27c6291ee32',
+            'uploader_id': '67449889',
+            'upload_date': '20190322',
+            'timestamp': int,  # timestamp is unstable
+            'duration': 1082.0,
+            'view_count': int,
+            'comment_count': int,
+        },
     }]
 
     _VALID_URL = r'https?://(?:www\.|secure\.|sp\.)?nicovideo\.jp/watch/(?P<id>(?:[a-z]{2})?[0-9]+)'

From 6d4932f02347bb1d0228b20798435930022bf316 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Sun, 18 Apr 2021 01:46:40 +0100
Subject: [PATCH 463/860] Try for timestamp, description from
 window.__INITIAL_DATA__ pages

---
 youtube_dl/extractor/bbc.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 247d982ce..37d427a66 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -1205,7 +1205,10 @@ class BBCIE(BBCCoUkIE):
                 if name == 'media-experience':
                     parse_media(try_get(resp, lambda x: x['data']['initialItem']['mediaItem'], dict))
                 elif name == 'article':
-                    for block in (try_get(resp, lambda x: x['data']['blocks'], list) or []):
+                    for block in (try_get(resp,
+                                          (lambda x: x['data']['blocks'],
+                                           lambda x: x['data']['content']['model']['blocks'],),
+                                          list) or []):
                         if block.get('type') != 'media':
                             continue
                         parse_media(block.get('model'))

From 58babe9af79215bd6bdf07da0a8ebb1d3650e00b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 30 Nov 2021 05:15:33 +0000
Subject: [PATCH 464/860] Support __INITIAL_DATA__ with stringified JSON

Add test and fix test for bbcthreeConfig
---
 youtube_dl/extractor/bbc.py | 50 +++++++++++++++++++++++++++++--------
 1 file changed, 39 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 37d427a66..088af9823 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -12,6 +12,7 @@ from ..compat import (
     compat_HTTPError,
     compat_parse_qs,
     compat_str,
+    compat_urllib_error,
     compat_urllib_parse_urlparse,
     compat_urlparse,
 )
@@ -395,9 +396,17 @@ class BBCCoUkIE(InfoExtractor):
                         formats.extend(self._extract_mpd_formats(
                             href, programme_id, mpd_id=format_id, fatal=False))
                     elif transfer_format == 'hls':
-                        formats.extend(self._extract_m3u8_formats(
-                            href, programme_id, ext='mp4', entry_protocol='m3u8_native',
-                            m3u8_id=format_id, fatal=False))
+                        # TODO: let expected_status be passed into _extract_xxx_formats() instead
+                        try:
+                            fmts = self._extract_m3u8_formats(
+                                href, programme_id, ext='mp4', entry_protocol='m3u8_native',
+                                m3u8_id=format_id, fatal=False)
+                        except ExtractorError as e:
+                            if not (isinstance(e.exc_info[1], compat_urllib_error.HTTPError)
+                                    and e.exc_info[1].code in (403, 404)):
+                                raise
+                            fmts = []
+                        formats.extend(fmts)
                     elif transfer_format == 'hds':
                         formats.extend(self._extract_f4m_formats(
                             href, programme_id, f4m_id=format_id, fatal=False))
@@ -775,21 +784,33 @@ class BBCIE(BBCCoUkIE):
             'timestamp': 1437785037,
             'upload_date': '20150725',
         },
+    }, {
+        # video with window.__INITIAL_DATA__ and value as JSON string
+        'url': 'https://www.bbc.com/news/av/world-europe-59468682',
+        'info_dict': {
+            'id': 'p0b71qth',
+            'ext': 'mp4',
+            'title': 'Why France is making this woman a national hero',
+            'description': 'md5:7affdfab80e9c3a1f976230a1ff4d5e4',
+            'thumbnail': r're:https?://.+/.+\.jpg',
+            'timestamp': 1638230731,
+            'upload_date': '20211130',
+        },
     }, {
         # single video article embedded with data-media-vpid
         'url': 'http://www.bbc.co.uk/sport/rowing/35908187',
         'only_matching': True,
     }, {
+        # bbcthreeConfig
         'url': 'https://www.bbc.co.uk/bbcthree/clip/73d0bbd0-abc3-4cea-b3c0-cdae21905eb1',
         'info_dict': {
             'id': 'p06556y7',
             'ext': 'mp4',
-            'title': 'Transfers: Cristiano Ronaldo to Man Utd, Arsenal to spend?',
-            'description': 'md5:4b7dfd063d5a789a1512e99662be3ddd',
+            'title': 'Things Not To Say to people that live on council estates',
+            'description': "From being labelled a 'chav', to the presumption that they're 'scroungers', people who live on council estates encounter all kinds of prejudices and false assumptions about themselves, their families, and their lifestyles. Here, eight people discuss the common statements, misconceptions, and clichés that they're tired of hearing.",
+            'duration': 360,
+            'thumbnail': r're:https?://.+/.+\.jpg',
         },
-        'params': {
-            'skip_download': True,
-        }
     }, {
         # window.__PRELOADED_STATE__
         'url': 'https://www.bbc.co.uk/radio/play/b0b9z4yl',
@@ -1162,9 +1183,16 @@ class BBCIE(BBCCoUkIE):
                 return self.playlist_result(
                     entries, playlist_id, playlist_title, playlist_description)
 
-        initial_data = self._parse_json(self._search_regex(
-            r'window\.__INITIAL_DATA__\s*=\s*({.+?});', webpage,
-            'preload state', default='{}'), playlist_id, fatal=False)
+        initial_data = self._search_regex(
+            r'window\.__INITIAL_DATA__\s*=\s*("{.+?}")\s*;', webpage,
+            'quoted preload state', default=None)
+        if initial_data is None:
+            initial_data = self._search_regex(
+                r'window\.__INITIAL_DATA__\s*=\s*({.+?})\s*;', webpage,
+                'preload state', default={})
+        else:
+            initial_data = self._parse_json(initial_data or '"{}"', playlist_id, fatal=False)
+        initial_data = self._parse_json(initial_data, playlist_id, fatal=False)
         if initial_data:
             def parse_media(media):
                 if not media:

From c820a284a23438f065171b7e222024d01893a95f Mon Sep 17 00:00:00 2001
From: Abdullah Ibn Fulan <ibnfulan@tutanota.de>
Date: Tue, 17 Aug 2021 18:22:07 +0600
Subject: [PATCH 465/860] [extractor/audiomack] Updated URL regex, corrected
 invalid testcases, fixed bug

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/audiomack.py | 40 ++++++++++++++++---------------
 1 file changed, 21 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index cc7771354..638eb4041 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -14,7 +14,7 @@ from ..utils import (
 
 
 class AudiomackIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?audiomack\.com/song/(?P<id>[\w/-]+)'
+    _VALID_URL = r'https?://(?:www\.)?audiomack\.com/(?:song/|(?=.+/song/))(?P<id>[\w/-]+)'
     IE_NAME = 'audiomack'
     _TESTS = [
         # hosted on audiomack
@@ -29,25 +29,27 @@ class AudiomackIE(InfoExtractor):
             }
         },
         # audiomack wrapper around soundcloud song
+        # Needs new test URL.
         {
             'add_ie': ['Soundcloud'],
             'url': 'http://www.audiomack.com/song/hip-hop-daily/black-mamba-freestyle',
-            'info_dict': {
-                'id': '258901379',
-                'ext': 'mp3',
-                'description': 'mamba day freestyle for the legend Kobe Bryant ',
-                'title': 'Black Mamba Freestyle [Prod. By Danny Wolf]',
-                'uploader': 'ILOVEMAKONNEN',
-                'upload_date': '20160414',
-            }
+            'only_matching': True,
+            # 'info_dict': {
+                # 'id': '258901379',
+                # 'ext': 'mp3',
+                # 'description': 'mamba day freestyle for the legend Kobe Bryant ',
+                # 'title': 'Black Mamba Freestyle [Prod. By Danny Wolf]',
+                # 'uploader': 'ILOVEMAKONNEN',
+                # 'upload_date': '20160414',
+            # }
         },
     ]
 
     def _real_extract(self, url):
-        # URLs end with [uploader name]/[uploader title]
+        # URLs end with [uploader name]/song/[uploader title]
         # this title is whatever the user types in, and is rarely
         # the proper song title.  Real metadata is in the api response
-        album_url_tag = self._match_id(url)
+        album_url_tag = self._match_id(url).replace('/song/', '/')
 
         # Request the extended version of the api for extra fields like artist and title
         api_response = self._download_json(
@@ -79,7 +81,7 @@ class AudiomackAlbumIE(InfoExtractor):
         # Standard album playlist
         {
             'url': 'http://www.audiomack.com/album/flytunezcom/tha-tour-part-2-mixtape',
-            'playlist_count': 15,
+            'playlist_count': 11,
             'info_dict':
             {
                 'id': '812251',
@@ -95,24 +97,24 @@ class AudiomackAlbumIE(InfoExtractor):
             },
             'playlist': [{
                 'info_dict': {
-                    'title': 'PPP (Pistol P Project) - 9. Heaven or Hell (CHIMACA) ft Zuse (prod by DJ FU)',
-                    'id': '837577',
+                    'title': 'PPP (Pistol P Project) - 10. 4 Minutes Of Hell Part 4 (prod by DY OF 808 MAFIA)',
+                    'id': '837580',
                     'ext': 'mp3',
                     'uploader': 'Lil Herb a.k.a. G Herbo',
                 }
             }],
             'params': {
-                'playliststart': 9,
-                'playlistend': 9,
+                'playliststart': 2,
+                'playlistend': 2,
             }
         }
     ]
 
     def _real_extract(self, url):
-        # URLs end with [uploader name]/[uploader title]
+        # URLs end with [uploader name]/album/[uploader title]
         # this title is whatever the user types in, and is rarely
         # the proper song title.  Real metadata is in the api response
-        album_url_tag = self._match_id(url)
+        album_url_tag = self._match_id(url).replace('/album/', '/')
         result = {'_type': 'playlist', 'entries': []}
         # There is no one endpoint for album metadata - instead it is included/repeated in each song's metadata
         # Therefore we don't know how many songs the album has and must infi-loop until failure
@@ -134,7 +136,7 @@ class AudiomackAlbumIE(InfoExtractor):
                 # Pull out the album metadata and add to result (if it exists)
                 for resultkey, apikey in [('id', 'album_id'), ('title', 'album_title')]:
                     if apikey in api_response and resultkey not in result:
-                        result[resultkey] = api_response[apikey]
+                        result[resultkey] = compat_str(api_response[apikey])
                 song_id = url_basename(api_response['url']).rpartition('.')[0]
                 result['entries'].append({
                     'id': compat_str(api_response.get('id', song_id)),

From 16a3fe2ba6b4c86e60bca930253c81c8efdd676b Mon Sep 17 00:00:00 2001
From: Abdullah Ibn Fulan <54185653+abdullah-if@users.noreply.github.com>
Date: Tue, 17 Aug 2021 19:56:39 +0000
Subject: [PATCH 466/860] Updated Album URL regex

Mistakenly forgot to edit a line in last commit.

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/audiomack.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/audiomack.py b/youtube_dl/extractor/audiomack.py
index 638eb4041..4d1fbad1f 100644
--- a/youtube_dl/extractor/audiomack.py
+++ b/youtube_dl/extractor/audiomack.py
@@ -75,7 +75,7 @@ class AudiomackIE(InfoExtractor):
 
 
 class AudiomackAlbumIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?audiomack\.com/album/(?P<id>[\w/-]+)'
+    _VALID_URL = r'https?://(?:www\.)?audiomack\.com/(?:album/|(?=.+/album/))(?P<id>[\w/-]+)'
     IE_NAME = 'audiomack:album'
     _TESTS = [
         # Standard album playlist

From ddc080a562cce984ac4a86969f511b1ae59421bf Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Mon, 18 Oct 2021 15:54:26 +0100
Subject: [PATCH 467/860] Add ArteTVCategoryIE to support category playlists

---
 youtube_dl/extractor/arte.py       | 47 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 48 insertions(+)

diff --git a/youtube_dl/extractor/arte.py b/youtube_dl/extractor/arte.py
index 03abdbfaf..5bfe57b10 100644
--- a/youtube_dl/extractor/arte.py
+++ b/youtube_dl/extractor/arte.py
@@ -12,6 +12,7 @@ from ..utils import (
     ExtractorError,
     int_or_none,
     qualities,
+    strip_or_none,
     try_get,
     unified_strdate,
     url_or_none,
@@ -252,3 +253,49 @@ class ArteTVPlaylistIE(ArteTVBaseIE):
         title = collection.get('title')
         description = collection.get('shortDescription') or collection.get('teaserText')
         return self.playlist_result(entries, playlist_id, title, description)
+
+
+class ArteTVCategoryIE(ArteTVBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?arte\.tv/(?P<lang>%s)/videos/(?P<id>[\w-]+(?:/[\w-]+)*)/?\s*$' % ArteTVBaseIE._ARTE_LANGUAGES
+    _TESTS = [{
+        'url': 'https://www.arte.tv/en/videos/politics-and-society/',
+        'info_dict': {
+            'id': 'politics-and-society',
+            'title': 'Politics and society',
+            'description': 'Investigative documentary series, geopolitical analysis, and international commentary',
+        },
+        'playlist_mincount': 13,
+    },
+    ]
+
+    @classmethod
+    def suitable(cls, url):
+        return (
+            not any(ie.suitable(url) for ie in (ArteTVIE, ArteTVPlaylistIE, ))
+            and super(ArteTVCategoryIE, cls).suitable(url))
+
+    def _real_extract(self, url):
+        lang, playlist_id = re.match(self._VALID_URL, url).groups()
+        webpage = self._download_webpage(url, playlist_id)
+
+        items = []
+        for video in re.finditer(
+                r'<a\b[^>]*?href\s*=\s*(?P<q>"|\'|\b)(?P<url>https?://www\.arte\.tv/%s/videos/[\w/-]+)(?P=q)' % lang,
+                webpage):
+            video = video.group('url')
+            if video == url:
+                continue
+            if any(ie.suitable(video) for ie in (ArteTVIE, ArteTVPlaylistIE, )):
+                items.append(video)
+
+        if items:
+            title = (self._og_search_title(webpage, default=None)
+                     or self._html_search_regex(r'<title\b[^>]*>([^<]+)</title>', default=None))
+            title = strip_or_none(title.rsplit('|', 1)[0]) or self._generic_title(url)
+
+            result = self.playlist_from_matches(items, playlist_id=playlist_id, playlist_title=title)
+            if result:
+                description = self._og_search_description(webpage, default=None)
+                if description:
+                    result['description'] = description
+                return result
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e70daf2b1..50b7cb4a0 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -71,6 +71,7 @@ from .arte import (
     ArteTVIE,
     ArteTVEmbedIE,
     ArteTVPlaylistIE,
+    ArteTVCategoryIE,
 )
 from .arnes import ArnesIE
 from .asiancrush import (

From 734dfbb4e3ad4ee4d98609dc902ac864b94033a4 Mon Sep 17 00:00:00 2001
From: Seonghyeon Cho <seonghyeoncho96@gmail.com>
Date: Wed, 13 Oct 2021 20:27:40 +0900
Subject: [PATCH 468/860] Remove redundant assigning `format_id`

---
 youtube_dl/extractor/uol.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/extractor/uol.py b/youtube_dl/extractor/uol.py
index 628adf219..59f8e5dc3 100644
--- a/youtube_dl/extractor/uol.py
+++ b/youtube_dl/extractor/uol.py
@@ -95,7 +95,6 @@ class UOLIE(InfoExtractor):
                 if v:
                     query[k] = v
             f_url = update_url_query(f_url, query)
-            format_id = format_id
             if format_id == 'HLS':
                 m3u8_formats = self._extract_m3u8_formats(
                     f_url, media_id, 'mp4', 'm3u8_native',

From 47b0c8697a39bbd64d5b922f81ad74ee4d2a3136 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 7 Feb 2022 13:28:21 +0000
Subject: [PATCH 469/860] [ARD] Back-port subtitle extraction from yt-dlp PR
 2409

Authored by: fstirlitz
Fixes #30543
Closes #17766 (thanks ngdio)
---
 youtube_dl/extractor/ard.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/ard.py b/youtube_dl/extractor/ard.py
index d45a9fe52..a5b1f54d5 100644
--- a/youtube_dl/extractor/ard.py
+++ b/youtube_dl/extractor/ard.py
@@ -332,9 +332,24 @@ class ARDIE(InfoExtractor):
             formats.append(f)
         self._sort_formats(formats)
 
+        _SUB_FORMATS = (
+            ('./dataTimedText', 'ttml'),
+            ('./dataTimedTextNoOffset', 'ttml'),
+            ('./dataTimedTextVtt', 'vtt'),
+        )
+
+        subtitles = {}
+        for subsel, subext in _SUB_FORMATS:
+            for node in video_node.findall(subsel):
+                subtitles.setdefault('de', []).append({
+                    'url': node.attrib['url'],
+                    'ext': subext,
+                })
+
         return {
             'id': xpath_text(video_node, './videoId', default=display_id),
             'formats': formats,
+            'subtitles': subtitles,
             'display_id': display_id,
             'title': video_node.find('./title').text,
             'duration': parse_duration(video_node.find('./duration').text),

From 825d3426c56aabfc91aea139f2e6e0589f8096bc Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 9 Feb 2022 02:40:34 +0000
Subject: [PATCH 470/860] [Nuvid] Use site JSON for video details (#29332)

Back-port yt-dlp PR 1022 onto PR #17890 and update

Video details aren't in the original HTML now but populated by async JS

Co-authored by: u-spec-png
Co-authored by: vidaritos
---
 youtube_dl/extractor/nuvid.py | 120 +++++++++++++++++++++++-----------
 1 file changed, 81 insertions(+), 39 deletions(-)

diff --git a/youtube_dl/extractor/nuvid.py b/youtube_dl/extractor/nuvid.py
index ab6bfcd7f..f6c94dd77 100644
--- a/youtube_dl/extractor/nuvid.py
+++ b/youtube_dl/extractor/nuvid.py
@@ -1,71 +1,113 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
-import re
-
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     parse_duration,
+    int_or_none,
+    try_get,
+    url_or_none,
 )
 
+import re
+
 
 class NuvidIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www|m)\.nuvid\.com/video/(?P<id>[0-9]+)'
-    _TEST = {
-        'url': 'http://m.nuvid.com/video/1310741/',
-        'md5': 'eab207b7ac4fccfb4e23c86201f11277',
+    _TESTS = [{
+        'url': 'https://www.nuvid.com/video/6513023/italian-babe',
+        'md5': '772d2f8288f3d3c5c45f7a41761c7844',
         'info_dict': {
-            'id': '1310741',
+            'id': '6513023',
             'ext': 'mp4',
-            'title': 'Horny babes show their awesome bodeis and',
-            'duration': 129,
+            'title': 'italian babe',
+            'format_id': '360p',
+            'duration': 321.0,
             'age_limit': 18,
+            'thumbnail': r're:https?://.+\.jpg',
+            'thumbnails': list,
         }
-    }
+    }, {
+        'url': 'https://m.nuvid.com/video/6523263',
+        'md5': 'ebd22ce8e47e1d9a4d0756a15c67da52',
+        'info_dict': {
+            'id': '6523263',
+            'ext': 'mp4',
+            'title': 'Slut brunette college student anal dorm',
+            'format_id': '720p',
+            'duration': 421.0,
+            'age_limit': 18,
+            'thumbnail': r're:https?://.+\.jpg',
+            'thumbnails': list,
+        }
+    }, {
+        'url': 'http://m.nuvid.com/video/6415801/',
+        'md5': '638d5ececb138d5753593f751ae3f697',
+        'info_dict': {
+            'id': '6415801',
+            'ext': 'mp4',
+            'title': 'My best friend wanted to fuck my wife for a long time',
+            'format_id': '720p',
+            'duration': 1882,
+            'age_limit': 18,
+            'thumbnail': r're:https?://.+\.jpg',
+            'thumbnails': list,
+        }
+    }]
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        page_url = 'http://m.nuvid.com/video/%s' % video_id
+        qualities = {
+            'lq': '360p',
+            'hq': '720p',
+        }
+
+        json_url = 'https://www.nuvid.com/player_config_json/?vid={video_id}&aid=0&domain_id=0&embed=0&check_speed=0'.format(**locals())
+        video_data = self._download_json(
+            json_url, video_id, headers={
+                'Accept': 'application/json, text/javascript, */*; q = 0.01',
+                'Content-Type': 'application/x-www-form-urlencoded; charset=utf-8',
+            }) or {}
+
+        # nice to have, not required
         webpage = self._download_webpage(
-            page_url, video_id, 'Downloading video page')
-        # When dwnld_speed exists and has a value larger than the MP4 file's
-        # bitrate, Nuvid returns the MP4 URL
-        # It's unit is 100bytes/millisecond, see mobile-nuvid-min.js for the algorithm
-        self._set_cookie('nuvid.com', 'dwnld_speed', '10.0')
-        mp4_webpage = self._download_webpage(
-            page_url, video_id, 'Downloading video page for MP4 format')
+            'http://m.nuvid.com/video/%s' % (video_id, ),
+            video_id, 'Downloading video page', fatal=False) or ''
+
+        title = (
+            try_get(video_data, lambda x: x['title'], compat_str)
+            or self._html_search_regex(
+                (r'''<span\s[^>]*?\btitle\s*=\s*(?P<q>"|'|\b)(?P<title>[^"]+)(?P=q)\s*>''',
+                 r'''<div\s[^>]*?\bclass\s*=\s*(?P<q>"|'|\b)thumb-holder video(?P=q)>\s*<h5\b[^>]*>(?P<title>[^<]+)</h5''',
+                 r'''<span\s[^>]*?\bclass\s*=\s*(?P<q>"|'|\b)title_thumb(?P=q)>(?P<title>[^<]+)</span'''),
+                webpage, 'title', group='title')).strip()
 
-        html5_video_re = r'(?s)<(?:video|audio)[^<]*(?:>.*?<source[^>]*)?\s+src=["\'](.*?)["\']',
-        video_url = self._html_search_regex(html5_video_re, webpage, video_id)
-        mp4_video_url = self._html_search_regex(html5_video_re, mp4_webpage, video_id)
         formats = [{
-            'url': video_url,
-        }]
-        if mp4_video_url != video_url:
-            formats.append({
-                'url': mp4_video_url,
-            })
+            'url': source,
+            'format_id': qualities.get(quality),
+            'height': int_or_none(qualities.get(quality)[:-1]),
+        } for quality, source in video_data.get('files').items() if source]
 
-        title = self._html_search_regex(
-            [r'<span title="([^"]+)">',
-             r'<div class="thumb-holder video">\s*<h5[^>]*>([^<]+)</h5>',
-             r'<span[^>]+class="title_thumb">([^<]+)</span>'], webpage, 'title').strip()
+        self._check_formats(formats, video_id)
+        self._sort_formats(formats)
+
+        duration = parse_duration(video_data.get('duration') or video_data.get('duration_format'))
         thumbnails = [
-            {
-                'url': thumb_url,
-            } for thumb_url in re.findall(r'<img src="([^"]+)" alt="" />', webpage)
+            {'url': thumb_url, }
+            for thumb_url in (
+                url_or_none(src) for src in re.findall(
+                    r'<div\s+class\s*=\s*"video-tmb-wrap"\s*>\s*<img\s+src\s*=\s*"([^"]+)"\s*/>',
+                    webpage))
         ]
-        thumbnail = thumbnails[0]['url'] if thumbnails else None
-        duration = parse_duration(self._html_search_regex(
-            [r'<i class="fa fa-clock-o"></i>\s*(\d{2}:\d{2})',
-             r'<span[^>]+class="view_time">([^<]+)</span>'], webpage, 'duration', fatal=False))
 
         return {
             'id': video_id,
+            'formats': formats,
             'title': title,
+            'thumbnail': url_or_none(video_data.get('poster')),
             'thumbnails': thumbnails,
-            'thumbnail': thumbnail,
             'duration': duration,
             'age_limit': 18,
-            'formats': formats,
         }

From 266b6ef18520f8de60fa143e154e4b12be12afb7 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 9 Feb 2022 21:21:59 +0000
Subject: [PATCH 471/860] [BBC] Also allow PID with leading 'l' (live?)

---
 youtube_dl/extractor/bbc.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bbc.py b/youtube_dl/extractor/bbc.py
index 088af9823..378b52f4f 100644
--- a/youtube_dl/extractor/bbc.py
+++ b/youtube_dl/extractor/bbc.py
@@ -40,7 +40,7 @@ from ..utils import (
 class BBCCoUkIE(InfoExtractor):
     IE_NAME = 'bbc.co.uk'
     IE_DESC = 'BBC iPlayer'
-    _ID_REGEX = r'(?:[pbm][\da-z]{7}|w[\da-z]{7,14})'
+    _ID_REGEX = r'(?:[pbml][\da-z]{7}|w[\da-z]{7,14})'
     _VALID_URL = r'''(?x)
                     https?://
                         (?:www\.)?bbc\.co\.uk/

From 8ff961d10faed848009f9e2ec03fa390b486694d Mon Sep 17 00:00:00 2001
From: kikuyan <kikuyan@users.noreply.github.com>
Date: Thu, 23 Dec 2021 11:40:45 +0900
Subject: [PATCH 472/860] [extractor/videa] fix extraction in Py2

Fixes #30416
---
 youtube_dl/extractor/videa.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/videa.py b/youtube_dl/extractor/videa.py
index ab2c15cde..bdb95891d 100644
--- a/youtube_dl/extractor/videa.py
+++ b/youtube_dl/extractor/videa.py
@@ -91,7 +91,7 @@ class VideaIE(InfoExtractor):
             k = S[(S[i] + S[j]) % 256]
             res += compat_struct_pack('B', k ^ compat_ord(cipher_text[m]))
 
-        return res.decode()
+        return res.decode('utf-8')
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
@@ -121,7 +121,7 @@ class VideaIE(InfoExtractor):
                 compat_b64decode(b64_info), key), video_id)
 
         video = xpath_element(info, './video', 'video')
-        if not video:
+        if video is None:
             raise ExtractorError(xpath_element(
                 info, './error', fatal=True), expected=True)
         sources = xpath_element(

From 74f8cc48afa59e1a125f939c060b21654d29789c Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 9 Feb 2022 04:37:28 +0000
Subject: [PATCH 473/860] [extractor/videa] Back-port from yt-dlp PRs 463+1028

Authored by: nyuszika7h
---
 youtube_dl/extractor/videa.py | 53 ++++++++++++++++++++++++++---------
 1 file changed, 39 insertions(+), 14 deletions(-)

diff --git a/youtube_dl/extractor/videa.py b/youtube_dl/extractor/videa.py
index bdb95891d..4589e78a1 100644
--- a/youtube_dl/extractor/videa.py
+++ b/youtube_dl/extractor/videa.py
@@ -12,6 +12,7 @@ from ..utils import (
     mimetype2ext,
     parse_codecs,
     update_url_query,
+    urljoin,
     xpath_element,
     xpath_text,
 )
@@ -19,6 +20,7 @@ from ..compat import (
     compat_b64decode,
     compat_ord,
     compat_struct_pack,
+    compat_urlparse,
 )
 
 
@@ -45,10 +47,24 @@ class VideaIE(InfoExtractor):
         },
     }, {
         'url': 'http://videa.hu/videok/origo/jarmuvek/supercars-elozes-jAHDWfWSJH5XuFhH',
-        'only_matching': True,
+        'md5': 'd57ccd8812c7fd491d33b1eab8c99975',
+        'info_dict': {
+            'id': 'jAHDWfWSJH5XuFhH',
+            'ext': 'mp4',
+            'title': 'Supercars előzés',
+            'thumbnail': r're:^https?://.*',
+            'duration': 64,
+        },
     }, {
         'url': 'http://videa.hu/player?v=8YfIAjxwWGwT8HVQ',
-        'only_matching': True,
+        'md5': '97a7af41faeaffd9f1fc864a7c7e7603',
+        'info_dict': {
+            'id': '8YfIAjxwWGwT8HVQ',
+            'ext': 'mp4',
+            'title': 'Az őrült kígyász 285 kígyót enged szabadon',
+            'thumbnail': r're:^https?://.*',
+            'duration': 21,
+        },
     }, {
         'url': 'http://videa.hu/player/v/8YfIAjxwWGwT8HVQ?autoplay=1',
         'only_matching': True,
@@ -95,9 +111,16 @@ class VideaIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        query = {'v': video_id}
-        player_page = self._download_webpage(
-            'https://videa.hu/player', video_id, query=query)
+        video_page = self._download_webpage(url, video_id)
+
+        if 'videa.hu/player' in url:
+            player_url = url
+            player_page = video_page
+        else:
+            player_url = self._search_regex(
+                r'<iframe.*?src="(/player\?[^"]+)"', video_page, 'player url')
+            player_url = urljoin(url, player_url)
+            player_page = self._download_webpage(player_url, video_id)
 
         nonce = self._search_regex(
             r'_xt\s*=\s*"([^"]+)"', player_page, 'nonce')
@@ -107,6 +130,7 @@ class VideaIE(InfoExtractor):
         for i in range(0, 32):
             result += s[i - (self._STATIC_SECRET.index(l[i]) - 31)]
 
+        query = compat_urlparse.parse_qs(compat_urlparse.urlparse(player_url).query)
         random_seed = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(8))
         query['_s'] = random_seed
         query['_t'] = result[:16]
@@ -127,7 +151,7 @@ class VideaIE(InfoExtractor):
         sources = xpath_element(
             info, './video_sources', 'sources', fatal=True)
         hash_values = xpath_element(
-            info, './hash_values', 'hash values', fatal=True)
+            info, './hash_values', 'hash values', fatal=False)
 
         title = xpath_text(video, './title', fatal=True)
 
@@ -136,15 +160,16 @@ class VideaIE(InfoExtractor):
             source_url = source.text
             source_name = source.get('name')
             source_exp = source.get('exp')
-            if not (source_url and source_name and source_exp):
+            if not (source_url and source_name):
                 continue
-            hash_value = xpath_text(hash_values, 'hash_value_' + source_name)
-            if not hash_value:
-                continue
-            source_url = update_url_query(source_url, {
-                'md5': hash_value,
-                'expires': source_exp,
-            })
+            hash_value = (
+                xpath_text(hash_values, 'hash_value_' + source_name)
+                if hash_values is not None else None)
+            if hash_value and source_exp:
+                source_url = update_url_query(source_url, {
+                    'md5': hash_value,
+                    'expires': source_exp,
+                })
             f = parse_codecs(source.get('codecs'))
             f.update({
                 'url': self._proto_relative_url(source_url),

From 29f7bfc4d7a80cecd67c19c25134481fbba6e175 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Petr=20Van=C4=9Bk?= <arkamar@atlas.cz>
Date: Tue, 11 Jan 2022 17:56:18 +0100
Subject: [PATCH 474/860] [streamcz] cherry-pick from yt-dlp

Cherry-picked-from: 7d449fff5346 ("[streamcz] Fix extractor (#1616)")
---
 youtube_dl/extractor/streamcz.py | 157 ++++++++++++++++---------------
 1 file changed, 80 insertions(+), 77 deletions(-)

diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index 58e0b4c80..0191c77de 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -1,105 +1,108 @@
 # coding: utf-8
-from __future__ import unicode_literals
-
-import hashlib
-import time
+import json
 
 from .common import InfoExtractor
 from ..utils import (
+    float_or_none,
     int_or_none,
-    sanitized_Request,
+    parse_codecs,
+    traverse_obj,
+    urljoin,
 )
 
 
-def _get_api_key(api_path):
-    if api_path.endswith('?'):
-        api_path = api_path[:-1]
-
-    api_key = 'fb5f58a820353bd7095de526253c14fd'
-    a = '{0:}{1:}{2:}'.format(api_key, api_path, int(round(time.time() / 24 / 3600)))
-    return hashlib.md5(a.encode('ascii')).hexdigest()
-
-
 class StreamCZIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?stream\.cz/.+/(?P<id>[0-9]+)'
-    _API_URL = 'http://www.stream.cz/API'
-
+    _VALID_URL = r'https?://(?:www\.)?(?:stream|televizeseznam)\.cz/[^?#]+/(?P<display_id>[^?#]+)-(?P<id>[0-9]+)'
     _TESTS = [{
-        'url': 'http://www.stream.cz/peklonataliri/765767-ecka-pro-deti',
-        'md5': '934bb6a6d220d99c010783c9719960d5',
+        'url': 'https://www.televizeseznam.cz/video/lajna/buh-57953890',
+        'md5': '40c41ade1464a390a0b447e333df4239',
         'info_dict': {
-            'id': '765767',
+            'id': '57953890',
             'ext': 'mp4',
-            'title': 'Peklo na talíři: Éčka pro děti',
-            'description': 'Taška s grónskou pomazánkou a další pekelnosti ZDE',
-            'thumbnail': 're:^http://im.stream.cz/episode/52961d7e19d423f8f06f0100',
-            'duration': 256,
-        },
+            'title': 'Bůh',
+            'display_id': 'buh',
+            'description': 'md5:8f5f09b9b7bc67df910486cdd88f7165',
+        }
     }, {
-        'url': 'http://www.stream.cz/blanik/10002447-tri-roky-pro-mazanka',
-        'md5': '849a88c1e1ca47d41403c2ba5e59e261',
+        'url': 'https://www.stream.cz/tajemno/znicehonic-jim-skrz-strechu-prolitnul-zahadny-predmet-badatele-vse-objasnili-64147267',
+        'md5': '3ee4d0be040e8f4a543e67e509d55e3f',
         'info_dict': {
-            'id': '10002447',
+            'id': '64147267',
             'ext': 'mp4',
-            'title': 'Kancelář Blaník: Tři roky pro Mazánka',
-            'description': 'md5:3862a00ba7bf0b3e44806b544032c859',
-            'thumbnail': 're:^http://im.stream.cz/episode/537f838c50c11f8d21320000',
-            'duration': 368,
-        },
+            'title': 'Zničehonic jim skrz střechu prolítnul záhadný předmět. Badatelé vše objasnili',
+            'display_id': 'znicehonic-jim-skrz-strechu-prolitnul-zahadny-predmet-badatele-vse-objasnili',
+            'description': 'md5:1dcb5e010eb697dedc5942f76c5b3744',
+        }
     }]
 
+    def _extract_formats(self, spl_url, video):
+        for ext, pref, streams in (
+                ('ts', -1, traverse_obj(video, ('http_stream', 'qualities'))),
+                ('mp4', 1, video.get('mp4'))):
+            for format_id, stream in streams.items():
+                if not stream.get('url'):
+                    continue
+                yield {
+                    'format_id': f'{format_id}-{ext}',
+                    'ext': ext,
+                    'source_preference': pref,
+                    'url': urljoin(spl_url, stream['url']),
+                    'tbr': float_or_none(stream.get('bandwidth'), scale=1000),
+                    'duration': float_or_none(stream.get('duration'), scale=1000),
+                    'width': traverse_obj(stream, ('resolution', 0)),
+                    'height': traverse_obj(stream, ('resolution', 1)) or int_or_none(format_id.replace('p', '')),
+                    **parse_codecs(stream.get('codec')),
+                }
+
     def _real_extract(self, url):
-        video_id = self._match_id(url)
-        api_path = '/episode/%s' % video_id
+        display_id, video_id = self._match_valid_url(url).groups()
 
-        req = sanitized_Request(self._API_URL + api_path)
-        req.add_header('Api-Password', _get_api_key(api_path))
-        data = self._download_json(req, video_id)
+        data = self._download_json(
+            'https://www.televizeseznam.cz/api/graphql', video_id, 'Downloading GraphQL result',
+            data=json.dumps({
+                'variables': {'urlName': video_id},
+                'query': '''
+                    query LoadEpisode($urlName : String){ episode(urlName: $urlName){ ...VideoDetailFragmentOnEpisode } }
+                    fragment VideoDetailFragmentOnEpisode on Episode {
+                        id
+                        spl
+                        urlName
+                        name
+                        perex
+                        duration
+                        views
+                    }'''
+            }).encode('utf-8'),
+            headers={'Content-Type': 'application/json;charset=UTF-8'}
+        )['data']['episode']
 
-        formats = []
-        for quality, video in enumerate(data['video_qualities']):
-            for f in video['formats']:
-                typ = f['type'].partition('/')[2]
-                qlabel = video.get('quality_label')
-                formats.append({
-                    'format_note': '%s-%s' % (qlabel, typ) if qlabel else typ,
-                    'format_id': '%s-%s' % (typ, f['quality']),
-                    'url': f['source'],
-                    'height': int_or_none(f['quality'].rstrip('p')),
-                    'quality': quality,
-                })
-        self._sort_formats(formats)
-
-        image = data.get('image')
-        if image:
-            thumbnail = self._proto_relative_url(
-                image.replace('{width}', '1240').replace('{height}', '697'),
-                scheme='http:',
-            )
-        else:
-            thumbnail = None
-
-        stream = data.get('_embedded', {}).get('stream:show', {}).get('name')
-        if stream:
-            title = '%s: %s' % (stream, data['name'])
-        else:
-            title = data['name']
+        spl_url = data['spl'] + 'spl2,3'
+        metadata = self._download_json(spl_url, video_id, 'Downloading playlist')
+        if 'Location' in metadata and 'data' not in metadata:
+            spl_url = metadata['Location']
+            metadata = self._download_json(spl_url, video_id, 'Downloading redirected playlist')
+        video = metadata['data']
 
         subtitles = {}
-        srt_url = data.get('subtitles_srt')
-        if srt_url:
-            subtitles['cs'] = [{
-                'ext': 'srt',
-                'url': srt_url,
-            }]
+        for subs in video.get('subtitles', {}).values():
+            if not subs.get('language'):
+                continue
+            for ext, sub_url in subs.get('urls').items():
+                subtitles.setdefault(subs['language'], []).append({
+                    'ext': ext,
+                    'url': urljoin(spl_url, sub_url)
+                })
+
+        formats = list(self._extract_formats(spl_url, video))
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
-            'title': title,
-            'thumbnail': thumbnail,
-            'formats': formats,
-            'description': data.get('web_site_text'),
-            'duration': int_or_none(data.get('duration')),
+            'display_id': display_id,
+            'title': data.get('name'),
+            'description': data.get('perex'),
+            'duration': float_or_none(data.get('duration')),
             'view_count': int_or_none(data.get('views')),
+            'formats': formats,
             'subtitles': subtitles,
         }

From 8088ce036ac4ce282f8f864c6b5f4f3987647221 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Petr=20Van=C4=9Bk?= <arkamar@atlas.cz>
Date: Sat, 12 Feb 2022 11:55:13 +0100
Subject: [PATCH 475/860] revert: use _match_valid_url function

---
 youtube_dl/extractor/streamcz.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index 0191c77de..998342e93 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -1,5 +1,6 @@
 # coding: utf-8
 import json
+import re
 
 from .common import InfoExtractor
 from ..utils import (
@@ -55,7 +56,7 @@ class StreamCZIE(InfoExtractor):
                 }
 
     def _real_extract(self, url):
-        display_id, video_id = self._match_valid_url(url).groups()
+        display_id, video_id = re.match(self._VALID_URL, url).groups()
 
         data = self._download_json(
             'https://www.televizeseznam.cz/api/graphql', video_id, 'Downloading GraphQL result',

From b1297308fb7b423a60c3a28c74ac014d7b385a2a Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Petr=20Van=C4=9Bk?= <arkamar@atlas.cz>
Date: Sat, 12 Feb 2022 12:28:30 +0100
Subject: [PATCH 476/860] avoid traverse_obj function

---
 youtube_dl/extractor/streamcz.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index 998342e93..fbdc44505 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -7,7 +7,6 @@ from ..utils import (
     float_or_none,
     int_or_none,
     parse_codecs,
-    traverse_obj,
     urljoin,
 )
 
@@ -38,7 +37,7 @@ class StreamCZIE(InfoExtractor):
 
     def _extract_formats(self, spl_url, video):
         for ext, pref, streams in (
-                ('ts', -1, traverse_obj(video, ('http_stream', 'qualities'))),
+                ('ts', -1, video.get('http_stream', {}).get('qualities', {})),
                 ('mp4', 1, video.get('mp4'))):
             for format_id, stream in streams.items():
                 if not stream.get('url'):
@@ -50,8 +49,8 @@ class StreamCZIE(InfoExtractor):
                     'url': urljoin(spl_url, stream['url']),
                     'tbr': float_or_none(stream.get('bandwidth'), scale=1000),
                     'duration': float_or_none(stream.get('duration'), scale=1000),
-                    'width': traverse_obj(stream, ('resolution', 0)),
-                    'height': traverse_obj(stream, ('resolution', 1)) or int_or_none(format_id.replace('p', '')),
+                    'width': stream.get('resolution', 2 * [0])[0] or None,
+                    'height': stream.get('resolution', 2 * [0])[1] or int_or_none(format_id.replace('p', '')),
                     **parse_codecs(stream.get('codec')),
                 }
 

From d02064218be76eba6350a13ccbbc473b1b439570 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Petr=20Van=C4=9Bk?= <arkamar@atlas.cz>
Date: Sat, 12 Feb 2022 12:30:29 +0100
Subject: [PATCH 477/860] do not use f-strings

---
 youtube_dl/extractor/streamcz.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index fbdc44505..d1736c023 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -43,7 +43,7 @@ class StreamCZIE(InfoExtractor):
                 if not stream.get('url'):
                     continue
                 yield {
-                    'format_id': f'{format_id}-{ext}',
+                    'format_id': '{}-{}'.format(format_id, ext),
                     'ext': ext,
                     'source_preference': pref,
                     'url': urljoin(spl_url, stream['url']),

From d8adca1b664fceb07f2b28b55c7e1855407296ea Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Petr=20Van=C4=9Bk?= <arkamar@atlas.cz>
Date: Sat, 12 Feb 2022 13:13:20 +0100
Subject: [PATCH 478/860] [streamcz] test fixes and one additional test

---
 youtube_dl/extractor/streamcz.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index d1736c023..60e770448 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -22,6 +22,20 @@ class StreamCZIE(InfoExtractor):
             'title': 'Bůh',
             'display_id': 'buh',
             'description': 'md5:8f5f09b9b7bc67df910486cdd88f7165',
+            'duration': 1369.6,
+            'view_count': int,
+        }
+    }, {
+        'url': 'https://www.stream.cz/kdo-to-mluvi/kdo-to-mluvi-velke-odhaleni-prinasi-novy-porad-uz-od-25-srpna-64087937',
+        'md5': '41fd358000086a1ccdb068c77809b158',
+        'info_dict': {
+            'id': '64087937',
+            'ext': 'mp4',
+            'title': 'Kdo to mluví? Velké odhalení přináší nový pořad už od 25. srpna',
+            'display_id': 'kdo-to-mluvi-velke-odhaleni-prinasi-novy-porad-uz-od-25-srpna',
+            'description': 'md5:97a811000a6460266029d6c1c2ebcd59',
+            'duration': 50.2,
+            'view_count': int,
         }
     }, {
         'url': 'https://www.stream.cz/tajemno/znicehonic-jim-skrz-strechu-prolitnul-zahadny-predmet-badatele-vse-objasnili-64147267',
@@ -31,7 +45,9 @@ class StreamCZIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Zničehonic jim skrz střechu prolítnul záhadný předmět. Badatelé vše objasnili',
             'display_id': 'znicehonic-jim-skrz-strechu-prolitnul-zahadny-predmet-badatele-vse-objasnili',
-            'description': 'md5:1dcb5e010eb697dedc5942f76c5b3744',
+            'description': 'md5:4b8ada6718d34bb011c4e04ca4bc19bf',
+            'duration': 442.84,
+            'view_count': int,
         }
     }]
 

From 85bf26c1d01f94b83476703e5c70022f01164ccf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Petr=20Van=C4=9Bk?= <arkamar@atlas.cz>
Date: Sat, 12 Feb 2022 15:02:08 +0100
Subject: [PATCH 479/860] resolve problem with unpacking operator for <py3.5

---
 youtube_dl/extractor/streamcz.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index 60e770448..179bdcaba 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -6,6 +6,7 @@ from .common import InfoExtractor
 from ..utils import (
     float_or_none,
     int_or_none,
+    merge_dicts,
     parse_codecs,
     urljoin,
 )
@@ -58,7 +59,7 @@ class StreamCZIE(InfoExtractor):
             for format_id, stream in streams.items():
                 if not stream.get('url'):
                     continue
-                yield {
+                yield merge_dicts({
                     'format_id': '{}-{}'.format(format_id, ext),
                     'ext': ext,
                     'source_preference': pref,
@@ -67,8 +68,7 @@ class StreamCZIE(InfoExtractor):
                     'duration': float_or_none(stream.get('duration'), scale=1000),
                     'width': stream.get('resolution', 2 * [0])[0] or None,
                     'height': stream.get('resolution', 2 * [0])[1] or int_or_none(format_id.replace('p', '')),
-                    **parse_codecs(stream.get('codec')),
-                }
+                }, parse_codecs(stream.get('codec')))
 
     def _real_extract(self, url):
         display_id, video_id = re.match(self._VALID_URL, url).groups()

From bf23bc0489cf304b2a8ab756f2f63b2cfa5586fe Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Petr=20Van=C4=9Bk?= <arkamar@atlas.cz>
Date: Sat, 12 Feb 2022 15:27:10 +0100
Subject: [PATCH 480/860] add missing __future__ import unicode_literals

---
 youtube_dl/extractor/streamcz.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index 179bdcaba..060ba32e0 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -1,4 +1,6 @@
 # coding: utf-8
+from __future__ import unicode_literals
+
 import json
 import re
 

From 34722270741fb9c06f978861c1e5f503291070d8 Mon Sep 17 00:00:00 2001
From: Vladimir Stavrinov <9163352+vstavrinov@users.noreply.github.com>
Date: Mon, 14 Feb 2022 20:54:31 +0300
Subject: [PATCH 481/860] [rutv] fix vbr for empty string value (#30623)

* [rutv] use str_to_int() (thx dirkf)
---
 youtube_dl/extractor/rutv.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/rutv.py b/youtube_dl/extractor/rutv.py
index d2713c19a..05f319396 100644
--- a/youtube_dl/extractor/rutv.py
+++ b/youtube_dl/extractor/rutv.py
@@ -6,7 +6,8 @@ import re
 from .common import InfoExtractor
 from ..utils import (
     ExtractorError,
-    int_or_none
+    int_or_none,
+    str_to_int
 )
 
 
@@ -179,7 +180,7 @@ class RUTVIE(InfoExtractor):
                         'player_url': 'http://player.rutv.ru/flash3v/osmf.swf?i=22',
                         'rtmp_live': True,
                         'ext': 'flv',
-                        'vbr': int(quality),
+                        'vbr': str_to_int(quality),
                         'preference': preference,
                     }
                 elif transport == 'm3u8':

From 782bfd26dbebea60e35f58ab18e218bedbecb782 Mon Sep 17 00:00:00 2001
From: "Lesmiscore (Naoya Ozaki)" <nao20010128@gmail.com>
Date: Thu, 24 Feb 2022 22:34:32 +0900
Subject: [PATCH 482/860] [bigo] add support for bigo.tv (#30635)

* [bigo] add support for bigo.tv

* [bigo] prepend "Bigo says"

* title fallback

* add error for invalid json data
---
 youtube_dl/extractor/bigo.py       | 59 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 60 insertions(+)
 create mode 100644 youtube_dl/extractor/bigo.py

diff --git a/youtube_dl/extractor/bigo.py b/youtube_dl/extractor/bigo.py
new file mode 100644
index 000000000..ddf76ac55
--- /dev/null
+++ b/youtube_dl/extractor/bigo.py
@@ -0,0 +1,59 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import ExtractorError, urlencode_postdata
+
+
+class BigoIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?bigo\.tv/(?:[a-z]{2,}/)?(?P<id>[^/]+)'
+
+    _TESTS = [{
+        'url': 'https://www.bigo.tv/ja/221338632',
+        'info_dict': {
+            'id': '6576287577575737440',
+            'title': '土よ〜💁‍♂️ 休憩室/REST room',
+            'thumbnail': r're:https?://.+',
+            'uploader': '✨Shin💫',
+            'uploader_id': '221338632',
+            'is_live': True,
+        },
+        'skip': 'livestream',
+    }, {
+        'url': 'https://www.bigo.tv/th/Tarlerm1304',
+        'only_matching': True,
+    }, {
+        'url': 'https://bigo.tv/115976881',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        user_id = self._match_id(url)
+
+        info_raw = self._download_json(
+            'https://bigo.tv/studio/getInternalStudioInfo',
+            user_id, data=urlencode_postdata({'siteId': user_id}))
+
+        if not isinstance(info_raw, dict):
+            raise ExtractorError('Received invalid JSON data')
+        if info_raw.get('code'):
+            raise ExtractorError(
+                'Bigo says: %s (code %s)' % (info_raw.get('msg'), info_raw.get('code')), expected=True)
+        info = info_raw.get('data') or {}
+
+        if not info.get('alive'):
+            raise ExtractorError('This user is offline.', expected=True)
+
+        return {
+            'id': info.get('roomId') or user_id,
+            'title': info.get('roomTopic') or info.get('nick_name') or user_id,
+            'formats': [{
+                'url': info.get('hls_src'),
+                'ext': 'mp4',
+                'protocol': 'm3u8',
+            }],
+            'thumbnail': info.get('snapshot'),
+            'uploader': info.get('nick_name'),
+            'uploader_id': user_id,
+            'is_live': True,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 50b7cb4a0..c73c4cd6c 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -115,6 +115,7 @@ from .bfmtv import (
 )
 from .bibeltv import BibelTVIE
 from .bigflix import BigflixIE
+from .bigo import BigoIE
 from .bild import BildIE
 from .bilibili import (
     BiliBiliIE,

From 923292ba643bf2a5c1fade797bd87a0de4f58d25 Mon Sep 17 00:00:00 2001
From: marieell <marieell@tuta.io>
Date: Thu, 10 Feb 2022 10:36:24 +0100
Subject: [PATCH 483/860] [aliexpress] Fix test case

---
 youtube_dl/extractor/aliexpress.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/aliexpress.py b/youtube_dl/extractor/aliexpress.py
index 6f241e683..9722fe9ac 100644
--- a/youtube_dl/extractor/aliexpress.py
+++ b/youtube_dl/extractor/aliexpress.py
@@ -18,7 +18,7 @@ class AliExpressLiveIE(InfoExtractor):
             'id': '2800002704436634',
             'ext': 'mp4',
             'title': 'CASIMA7.22',
-            'thumbnail': r're:http://.*\.jpg',
+            'thumbnail': r're:https?://.*\.jpg',
             'uploader': 'CASIMA Official Store',
             'timestamp': 1500717600,
             'upload_date': '20170722',

From 1f13ccfd7fcafbfd79ddd652967e02f9eda7ce79 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 24 Feb 2022 18:26:58 +0000
Subject: [PATCH 484/860] Fixed groups() call on potentially empty regex search
 object (#30676)

* Fixed groups() call on potentially empty regex search object.
- https://github.com/ytdl-org/youtube-dl/issues/30521

* minimising lines changed

Co-authored-by: yayorbitgum <50963144+yayorbitgum@users.noreply.github.com>
---
 youtube_dl/extractor/myspass.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/myspass.py b/youtube_dl/extractor/myspass.py
index db7ebc94c..f540c52ee 100644
--- a/youtube_dl/extractor/myspass.py
+++ b/youtube_dl/extractor/myspass.py
@@ -35,7 +35,9 @@ class MySpassIE(InfoExtractor):
         title = xpath_text(metadata, 'title', fatal=True)
         video_url = xpath_text(metadata, 'url_flv', 'download url', True)
         video_id_int = int(video_id)
-        for group in re.search(r'/myspass2009/\d+/(\d+)/(\d+)/(\d+)/', video_url).groups():
+
+        grps = re.search(r'/myspass2009/\d+/(\d+)/(\d+)/(\d+)/', video_url)
+        for group in grps.groups() if grps else []:
             group_int = int(group)
             if group_int > video_id_int:
                 video_url = video_url.replace(

From c4d1738316db45e03e0625650b3550334b66ab7f Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 24 Feb 2022 09:16:16 +0000
Subject: [PATCH 485/860] [CPAC] Add extractor for Canadian Parliament

CPACIE: single episode
CPACPlaylistIE: playlists and searches
---
 youtube_dl/extractor/cpac.py       | 148 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |   4 +
 2 files changed, 152 insertions(+)
 create mode 100644 youtube_dl/extractor/cpac.py

diff --git a/youtube_dl/extractor/cpac.py b/youtube_dl/extractor/cpac.py
new file mode 100644
index 000000000..22741152c
--- /dev/null
+++ b/youtube_dl/extractor/cpac.py
@@ -0,0 +1,148 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    int_or_none,
+    str_or_none,
+    try_get,
+    unified_timestamp,
+    update_url_query,
+    urljoin,
+)
+
+# compat_range
+try:
+    if callable(xrange):
+        range = xrange
+except (NameError, TypeError):
+    pass
+
+
+class CPACIE(InfoExtractor):
+    IE_NAME = 'cpac'
+    _VALID_URL = r'https?://(?:www\.)?cpac\.ca/(?P<fr>l-)?episode\?id=(?P<id>[\da-f]{8}(?:-[\da-f]{4}){3}-[\da-f]{12})'
+    _TEST = {
+        # 'url': 'http://www.cpac.ca/en/programs/primetime-politics/episodes/65490909',
+        'url': 'https://www.cpac.ca/episode?id=fc7edcae-4660-47e1-ba61-5b7f29a9db0f',
+        'md5': 'e46ad699caafd7aa6024279f2614e8fa',
+        'info_dict': {
+            'id': 'fc7edcae-4660-47e1-ba61-5b7f29a9db0f',
+            'ext': 'mp4',
+            'upload_date': '20220215',
+            'title': 'News Conference to Celebrate National Kindness Week – February 15, 2022',
+            'description': 'md5:466a206abd21f3a6f776cdef290c23fb',
+            'timestamp': 1644901200,
+        },
+        'params': {
+            'format': 'bestvideo',
+            'hls_prefer_native': True,
+        },
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        url_lang = 'fr' if '/l-episode?' in url else 'en'
+
+        content = self._download_json(
+            'https://www.cpac.ca/api/1/services/contentModel.json?url=/site/website/episode/index.xml&crafterSite=cpacca&id=' + video_id,
+            video_id)
+        video_url = try_get(content, lambda x: x['page']['details']['videoUrl'], compat_str)
+        formats = []
+        if video_url:
+            content = content['page']
+            title = str_or_none(content['details']['title_%s_t' % (url_lang, )])
+            formats = self._extract_m3u8_formats(video_url, video_id, m3u8_id='hls', ext='mp4')
+            for fmt in formats:
+                # prefer language to match URL
+                fmt_lang = fmt.get('language')
+                if fmt_lang == url_lang:
+                    fmt['language_preference'] = 10
+                elif not fmt_lang:
+                    fmt['language_preference'] = -1
+                else:
+                    fmt['language_preference'] = -10
+
+        self._sort_formats(formats)
+
+        category = str_or_none(content['details']['category_%s_t' % (url_lang, )])
+
+        def is_live(v_type):
+            return (v_type == 'live') if v_type is not None else None
+
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': title,
+            'description': str_or_none(content['details'].get('description_%s_t' % (url_lang, ))),
+            'timestamp': unified_timestamp(content['details'].get('liveDateTime')),
+            'category': [category] if category else None,
+            'thumbnail': urljoin(url, str_or_none(content['details'].get('image_%s_s' % (url_lang, )))),
+            'is_live': is_live(content['details'].get('type')),
+        }
+
+
+class CPACPlaylistIE(InfoExtractor):
+    IE_NAME = 'cpac:playlist'
+    _VALID_URL = r'(?i)https?://(?:www\.)?cpac\.ca/(?:program|search|(?P<fr>emission|rechercher))\?(?:[^&]+&)*?(?P<id>(?:id=\d+|programId=\d+|key=[^&]+))'
+
+    _TESTS = [{
+        'url': 'https://www.cpac.ca/program?id=6',
+        'info_dict': {
+            'id': 'id=6',
+            'title': 'Headline Politics',
+            'description': 'Watch CPAC’s signature long-form coverage of the day’s pressing political events as they unfold.',
+        },
+        'playlist_count': 10,
+    }, {
+        'url': 'https://www.cpac.ca/search?key=hudson&type=all&order=desc',
+        'info_dict': {
+            'id': 'key=hudson',
+            'title': 'hudson',
+        },
+        'playlist_count': 22,
+    }, {
+        'url': 'https://www.cpac.ca/search?programId=50',
+        'info_dict': {
+            'id': 'programId=50',
+            'title': '50',
+        },
+        'playlist_count': 9,
+    }, {
+        'url': 'https://www.cpac.ca/emission?id=6',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.cpac.ca/rechercher?key=hudson&type=all&order=desc',
+        'only_matching': True,
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        url_lang = 'fr' if any(x in url for x in ('/emission?', '/rechercher?')) else 'en'
+        pl_type, list_type = ('program', 'itemList') if any(x in url for x in ('/program?', '/emission?')) else ('search', 'searchResult')
+        api_url = (
+            'https://www.cpac.ca/api/1/services/contentModel.json?url=/site/website/%s/index.xml&crafterSite=cpacca&%s'
+            % (pl_type, video_id, ))
+        content = self._download_json(api_url, video_id)
+        entries = []
+        total_pages = int_or_none(try_get(content, lambda x: x['page'][list_type]['totalPages']), default=1)
+        for page in range(1, total_pages + 1):
+            if page > 1:
+                api_url = update_url_query(api_url, {'page': '%d' % (page, ), })
+                content = self._download_json(
+                    api_url, video_id,
+                    note='Downloading continuation - %d' % (page, ),
+                    fatal=False)
+
+            for item in try_get(content, lambda x: x['page'][list_type]['item'], list) or []:
+                episode_url = urljoin(url, try_get(item, lambda x: x['url_%s_s' % (url_lang, )]))
+                if episode_url:
+                    entries.append(episode_url)
+
+        return self.playlist_result(
+            (self.url_result(entry) for entry in entries),
+            playlist_id=video_id,
+            playlist_title=try_get(content, lambda x: x['page']['program']['title_%s_t' % (url_lang, )]) or video_id.split('=')[-1],
+            playlist_description=try_get(content, lambda x: x['page']['program']['description_%s_t' % (url_lang, )]),
+        )
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index c73c4cd6c..7c99cb7e0 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -255,6 +255,10 @@ from .commonprotocols import (
 from .condenast import CondeNastIE
 from .contv import CONtvIE
 from .corus import CorusIE
+from .cpac import (
+    CPACIE,
+    CPACPlaylistIE,
+)
 from .cracked import CrackedIE
 from .crackle import CrackleIE
 from .crooksandliars import CrooksAndLiarsIE

From f8e543c9063c1c7ad157936cb6a15b428ddb3896 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 7 Feb 2022 20:06:27 +0000
Subject: [PATCH 486/860] [Alsace20TV] Add new extractors Alsace20TVIE,
 Alsace20TVEmbedIE

---
 youtube_dl/extractor/alsace20tv.py | 89 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  4 ++
 2 files changed, 93 insertions(+)
 create mode 100644 youtube_dl/extractor/alsace20tv.py

diff --git a/youtube_dl/extractor/alsace20tv.py b/youtube_dl/extractor/alsace20tv.py
new file mode 100644
index 000000000..228cec3ec
--- /dev/null
+++ b/youtube_dl/extractor/alsace20tv.py
@@ -0,0 +1,89 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    dict_get,
+    get_element_by_class,
+    int_or_none,
+    unified_strdate,
+    url_or_none,
+)
+
+
+class Alsace20TVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?alsace20\.tv/(?:[\w-]+/)+[\w-]+-(?P<id>[\w]+)'
+    _TESTS = [{
+        'url': 'https://www.alsace20.tv/VOD/Actu/JT/Votre-JT-jeudi-3-fevrier-lyNHCXpYJh.html',
+        # 'md5': 'd91851bf9af73c0ad9b2cdf76c127fbb',
+        'info_dict': {
+            'id': 'lyNHCXpYJh',
+            'ext': 'mp4',
+            'description': 'md5:fc0bc4a0692d3d2dba4524053de4c7b7',
+            'title': 'Votre JT du jeudi 3 février',
+            'upload_date': '20220203',
+            'thumbnail': r're:https?://.+\.jpg',
+            'duration': 1073,
+            'view_count': int,
+        },
+        'params': {
+            'format': 'bestvideo',
+        },
+    }]
+
+    def _extract_video(self, video_id, url=None):
+        info = self._download_json(
+            'https://www.alsace20.tv/visionneuse/visio_v9_js.php?key=%s&habillage=0&mode=html' % (video_id, ),
+            video_id) or {}
+        title = info['titre']
+
+        formats = []
+        for res, fmt_url in (info.get('files') or {}).items():
+            formats.extend(
+                self._extract_smil_formats(fmt_url, video_id, fatal=False)
+                if '/smil:_' in fmt_url
+                else self._extract_mpd_formats(fmt_url, video_id, mpd_id=res, fatal=False))
+        self._sort_formats(formats)
+
+        webpage = (url and self._download_webpage(url, video_id, fatal=False)) or ''
+        thumbnail = url_or_none(dict_get(info, ('image', 'preview', )) or self._og_search_thumbnail(webpage))
+        upload_date = self._search_regex(r'/(\d{6})_', thumbnail, 'upload_date', default=None)
+        upload_date = unified_strdate('20%s-%s-%s' % (upload_date[:2], upload_date[2:4], upload_date[4:])) if upload_date else None
+        return {
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+            'description': clean_html(get_element_by_class('wysiwyg', webpage)),
+            'upload_date': upload_date,
+            'thumbnail': thumbnail,
+            'duration': int_or_none(self._og_search_property('video:duration', webpage) if webpage else None),
+            'view_count': int_or_none(info.get('nb_vues')),
+        }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        return self._extract_video(video_id, url)
+
+
+class Alsace20TVEmbedIE(Alsace20TVIE):
+    _VALID_URL = r'https?://(?:www\.)?alsace20\.tv/emb/(?P<id>[\w]+)'
+    _TESTS = [{
+        'url': 'https://www.alsace20.tv/emb/lyNHCXpYJh',
+        # 'md5': 'd91851bf9af73c0ad9b2cdf76c127fbb',
+        'info_dict': {
+            'id': 'lyNHCXpYJh',
+            'ext': 'mp4',
+            'title': 'Votre JT du jeudi 3 février',
+            'upload_date': '20220203',
+            'thumbnail': r're:https?://.+\.jpg',
+            'view_count': int,
+        },
+        'params': {
+            'format': 'bestvideo',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        return self._extract_video(video_id)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 7c99cb7e0..535080d0a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -51,6 +51,10 @@ from .anvato import AnvatoIE
 from .aol import AolIE
 from .allocine import AllocineIE
 from .aliexpress import AliExpressLiveIE
+from .alsace20tv import (
+    Alsace20TVIE,
+    Alsace20TVEmbedIE,
+)
 from .apa import APAIE
 from .aparat import AparatIE
 from .appleconnect import AppleConnectIE

From 4194d253c0b922addf0439228066cb4fb487bac3 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 30 Jul 2021 12:58:19 +0100
Subject: [PATCH 487/860] Avoid skipping ID when unlisted_hash is numeric

Pattern needed a non-greedy match; also replaced a redundant test with one for this, issue 29690
---
 youtube_dl/extractor/vimeo.py | 24 +++++++++++++++++++-----
 1 file changed, 19 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 0b386f450..a66912502 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -271,7 +271,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                         )?
                         vimeo(?:pro)?\.com/
                         (?!(?:channels|album|showcase)/[^/?#]+/?(?:$|[?#])|[^/]+/review/|ondemand/)
-                        (?:.*?/)?
+                        (?:.*?/)??
                         (?:
                             (?:
                                 play_redirect_hls|
@@ -517,14 +517,28 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'url': 'https://vimeo.com/7809605',
             'only_matching': True,
         },
-        {
-            'url': 'https://vimeo.com/160743502/abd0e13fb4',
-            'only_matching': True,
-        },
         {
             # requires passing unlisted_hash(a52724358e) to load_download_config request
             'url': 'https://vimeo.com/392479337/a52724358e',
             'only_matching': True,
+        },
+        {
+            # similar, but all numeric: ID must be 581039021, not 9603038895
+            # issue #29690
+            'url': 'https://vimeo.com/581039021/9603038895',
+            'info_dict': {
+                'id': '581039021',
+                # these have to be provided but we don't care
+                'ext': 'mp4',
+                'timestamp': 1627621014,
+                'title': 're:.+',
+                'uploader_id': 're:.+',
+                'uploader': 're:.+',
+                'upload_date': r're:\d+',
+            },
+            'params': {
+                'skip_download': True,
+            },
         }
         # https://gettingthingsdone.com/workflowmap/
         # vimeo embed with check-password page protected by Referer header

From 6508688e88c83bb811653083db9351702cd39a6a Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Sun, 1 Aug 2021 09:42:57 +0100
Subject: [PATCH 488/860] Make default upload_/release_date a compat_str

Ensures download tests pass in Python 2 as well as 3; also
add YoutubeDL tests for timestamp -> upload_date etc.
---
 test/test_YoutubeDL.py  | 19 +++++++++++++++++++
 youtube_dl/YoutubeDL.py |  2 +-
 2 files changed, 20 insertions(+), 1 deletion(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index a35effe0e..f8c8e619c 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -997,6 +997,25 @@ class TestYoutubeDL(unittest.TestCase):
         self.assertEqual(downloaded['extractor'], 'Video')
         self.assertEqual(downloaded['extractor_key'], 'Video')
 
+    def test_default_times(self):
+        """Test addition of missing upload/release/_date from /release_/timestamp"""
+        info = {
+            'id': '1234',
+            'url': TEST_URL,
+            'title': 'Title',
+            'ext': 'mp4',
+            'timestamp': 1631352900,
+            'release_timestamp': 1632995931,
+        }
+
+        params = {'simulate': True, }
+        ydl = FakeYDL(params)
+        out_info = ydl.process_ie_result(info)
+        self.assertTrue(isinstance(out_info['upload_date'], compat_str))
+        self.assertEqual(out_info['upload_date'], '20210911')
+        self.assertTrue(isinstance(out_info['release_date'], compat_str))
+        self.assertEqual(out_info['release_date'], '20210930')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index fe30758ef..69736acff 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1529,7 +1529,7 @@ class YoutubeDL(object):
                 # see http://bugs.python.org/issue1646728)
                 try:
                     upload_date = datetime.datetime.utcfromtimestamp(info_dict[ts_key])
-                    info_dict[date_key] = upload_date.strftime('%Y%m%d')
+                    info_dict[date_key] = compat_str(upload_date.strftime('%Y%m%d'))
                 except (ValueError, OverflowError, OSError):
                     pass
 

From 49c5293014bc11ec8c009856cd63cffa6296c1e1 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 22 Feb 2022 11:24:06 +0000
Subject: [PATCH 489/860] Ignore --external-downloader-args if
 --external-downloader was rejected

... and generate warning
---
 youtube_dl/YoutubeDL.py           | 11 ++++++++++-
 youtube_dl/downloader/__init__.py |  3 +++
 2 files changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 69736acff..019e309cb 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1906,8 +1906,17 @@ class YoutubeDL(object):
 
         if not self.params.get('skip_download', False):
             try:
+                def checked_get_suitable_downloader(info_dict, params):
+                    ed_args = params.get('external_downloader_args')
+                    dler = get_suitable_downloader(info_dict, params)
+                    if ed_args and not params.get('external_downloader_args'):
+                        # external_downloader_args was cleared because external_downloader was rejected
+                        self.report_warning('Requested external downloader cannot be used: '
+                                            'ignoring --external-downloader-args.')
+                    return dler
+
                 def dl(name, info):
-                    fd = get_suitable_downloader(info, self.params)(self, self.params)
+                    fd = checked_get_suitable_downloader(info, self.params)(self, self.params)
                     for ph in self._progress_hooks:
                         fd.add_progress_hook(ph)
                     if self.params.get('verbose'):
diff --git a/youtube_dl/downloader/__init__.py b/youtube_dl/downloader/__init__.py
index d8f2fa342..d701d6292 100644
--- a/youtube_dl/downloader/__init__.py
+++ b/youtube_dl/downloader/__init__.py
@@ -50,6 +50,9 @@ def _get_suitable_downloader(info_dict, params={}):
         ed = get_external_downloader(external_downloader)
         if ed.can_download(info_dict):
             return ed
+        # Avoid using unwanted args since external_downloader was rejected
+        if params.get('external_downloader_args'):
+            params['external_downloader_args'] = None
 
     protocol = info_dict['protocol']
     if protocol.startswith('m3u8') and info_dict.get('is_live'):

From 17d295a1ec6d04362740dd8a0c583690f5ba082a Mon Sep 17 00:00:00 2001
From: lihan7 <lihan7@xiaomi.com>
Date: Fri, 25 Mar 2022 15:46:28 +0800
Subject: [PATCH 490/860] [extractor/bilibili] Fix path "/audio/auxxxxx"
 download return 403

---
 youtube_dl/extractor/bilibili.py | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/youtube_dl/extractor/bilibili.py b/youtube_dl/extractor/bilibili.py
index bff6ea194..d42f0e98a 100644
--- a/youtube_dl/extractor/bilibili.py
+++ b/youtube_dl/extractor/bilibili.py
@@ -369,6 +369,11 @@ class BilibiliAudioIE(BilibiliAudioBaseIE):
             'filesize': int_or_none(play_data.get('size')),
         }]
 
+        for a_format in formats:
+            a_format.setdefault('http_headers', {}).update({
+                'Referer': url,
+            })
+
         song = self._call_api('song/info', au_id)
         title = song['title']
         statistic = song.get('statistic') or {}

From 9e5ca66f16998eb2a680e23a6e769e34001898c5 Mon Sep 17 00:00:00 2001
From: nixxo <nixxo@protonmail.com>
Date: Mon, 4 Jan 2021 15:11:47 +0100
Subject: [PATCH 491/860] [RAI] Added checks for DRM protected content (PR
 #27657)

reviewed by pukkandan (https://github.com/yt-dlp/yt-dlp/pull/150)
---
 youtube_dl/extractor/rai.py | 11 +++++++++++
 1 file changed, 11 insertions(+)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 67b86fc72..2abe164e0 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -158,6 +158,10 @@ class RaiPlayIE(RaiBaseIE):
         # subtitles at 'subtitlesArray' key (see #27698)
         'url': 'https://www.raiplay.it/video/2020/12/Report---04-01-2021-2e90f1de-8eee-4de4-ac0e-78d21db5b600.html',
         'only_matching': True,
+    }, {
+        # DRM protected
+        'url': 'https://www.raiplay.it/video/2020/09/Lo-straordinario-mondo-di-Zoey-S1E1-Lo-straordinario-potere-di-Zoey-ed493918-1d32-44b7-8454-862e473d00ff.html',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -166,6 +170,13 @@ class RaiPlayIE(RaiBaseIE):
         media = self._download_json(
             base + '.json', video_id, 'Downloading video JSON')
 
+        if try_get(
+                media,
+                (lambda x: x['rights_management']['rights']['drm'],
+                 lambda x: x['program_info']['rights_management']['rights']['drm']),
+                dict):
+            raise ExtractorError('This video is DRM protected.', expected=True)
+
         title = media['name']
 
         video = media['video']

From 1f50a07771fddb5f64617617d156bfdd593f951e Mon Sep 17 00:00:00 2001
From: nixxo <nixxo@protonmail.com>
Date: Wed, 27 Jan 2021 12:24:50 +0100
Subject: [PATCH 492/860] [RAI] Extend formats with direct http mp4 link (PR
 #27990)

* initial support for creating direct mp4 link
* improved regexes and info extraction
* added "connection: close" to request headers
* updated to https://github.com/yt-dlp/yt-dlp/pull/208
---
 youtube_dl/extractor/rai.py | 111 +++++++++++++++++++++++++++++++++++-
 1 file changed, 108 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 2abe164e0..7b0315a62 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -5,15 +5,16 @@ import re
 
 from .common import InfoExtractor
 from ..compat import (
-    compat_urlparse,
     compat_str,
+    compat_urlparse,
 )
 from ..utils import (
-    ExtractorError,
     determine_ext,
+    ExtractorError,
     find_xpath_attr,
     fix_xml_ampersands,
     GeoRestrictedError,
+    HEADRequest,
     int_or_none,
     parse_duration,
     remove_start,
@@ -96,12 +97,100 @@ class RaiBaseIE(InfoExtractor):
         if not formats and geoprotection is True:
             self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
 
+        formats.extend(self._create_http_urls(relinker_url, formats))
+
         return dict((k, v) for k, v in {
             'is_live': is_live,
             'duration': duration,
             'formats': formats,
         }.items() if v is not None)
 
+    def _create_http_urls(self, relinker_url, fmts):
+        _RELINKER_REG = r'https?://(?P<host>[^/]+?)/(?:i/)?(?P<extra>[^/]+?)/(?P<path>.+?)/(?P<id>\d+)(?:_(?P<quality>[\d\,]+))?(?:\.mp4|/playlist\.m3u8).+?'
+        _MP4_TMPL = '%s&overrideUserAgentRule=mp4-%s'
+        _QUALITY = {
+            # tbr: w, h
+            '250': [352, 198],
+            '400': [512, 288],
+            '700': [512, 288],
+            '800': [700, 394],
+            '1200': [736, 414],
+            '1800': [1024, 576],
+            '2400': [1280, 720],
+            '3200': [1440, 810],
+            '3600': [1440, 810],
+            '5000': [1920, 1080],
+            '10000': [1920, 1080],
+        }
+
+        def test_url(url):
+            resp = self._request_webpage(
+                HEADRequest(url), None, headers={'User-Agent': 'Rai'},
+                fatal=False, errnote=False, note=False)
+
+            if resp is False:
+                return False
+
+            if resp.code == 200:
+                return False if resp.url == url else resp.url
+            return None
+
+        def get_format_info(tbr):
+            import math
+            br = int_or_none(tbr)
+            if len(fmts) == 1 and not br:
+                br = fmts[0].get('tbr')
+            if br > 300:
+                tbr = compat_str(math.floor(br / 100) * 100)
+            else:
+                tbr = '250'
+
+            # try extracting info from available m3u8 formats
+            format_copy = None
+            for f in fmts:
+                if f.get('tbr'):
+                    br_limit = math.floor(br / 100)
+                    if br_limit - 1 <= math.floor(f['tbr'] / 100) <= br_limit + 1:
+                        format_copy = f.copy()
+            return {
+                'width': format_copy.get('width'),
+                'height': format_copy.get('height'),
+                'tbr': format_copy.get('tbr'),
+                'vcodec': format_copy.get('vcodec'),
+                'acodec': format_copy.get('acodec'),
+                'fps': format_copy.get('fps'),
+                'format_id': 'https-%s' % tbr,
+            } if format_copy else {
+                'width': _QUALITY[tbr][0],
+                'height': _QUALITY[tbr][1],
+                'format_id': 'https-%s' % tbr,
+                'tbr': int(tbr),
+            }
+
+        loc = test_url(_MP4_TMPL % (relinker_url, '*'))
+        if not isinstance(loc, compat_str):
+            return []
+
+        mobj = re.match(
+            _RELINKER_REG,
+            test_url(relinker_url) or '')
+        if not mobj:
+            return []
+
+        available_qualities = mobj.group('quality').split(',') if mobj.group('quality') else ['*']
+        available_qualities = [i for i in available_qualities if i]
+
+        formats = []
+        for q in available_qualities:
+            fmt = {
+                'url': _MP4_TMPL % (relinker_url, q),
+                'protocol': 'https',
+                'ext': 'mp4',
+            }
+            fmt.update(get_format_info(q))
+            formats.append(fmt)
+        return formats
+
     @staticmethod
     def _extract_subtitles(url, video_data):
         STL_EXT = 'stl'
@@ -151,6 +240,22 @@ class RaiPlayIE(RaiBaseIE):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # 1080p direct mp4 url
+        'url': 'https://www.raiplay.it/video/2021/03/Leonardo-S1E1-b5703b02-82ee-475a-85b6-c9e4a8adf642.html',
+        'md5': '2e501e8651d72f05ffe8f5d286ad560b',
+        'info_dict': {
+            'id': 'b5703b02-82ee-475a-85b6-c9e4a8adf642',
+            'ext': 'mp4',
+            'title': 'Leonardo - S1E1',
+            'alt_title': 'St 1 Ep 1 - Episodio 1',
+            'description': 'md5:f5360cd267d2de146e4e3879a5a47d31',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'uploader': 'Rai 1',
+            'duration': 3229,
+            'series': 'Leonardo',
+            'season': 'Season 1',
+        },
     }, {
         'url': 'http://www.raiplay.it/video/2016/11/gazebotraindesi-efebe701-969c-4593-92f3-285f0d1ce750.html?',
         'only_matching': True,
@@ -318,7 +423,7 @@ class RaiIE(RaiBaseIE):
     }, {
         # with ContentItem in og:url
         'url': 'http://www.rai.it/dl/RaiTV/programmi/media/ContentItem-efb17665-691c-45d5-a60c-5301333cbb0c.html',
-        'md5': '6865dd00cf0bbf5772fdd89d59bd768a',
+        'md5': '06345bd97c932f19ffb129973d07a020',
         'info_dict': {
             'id': 'efb17665-691c-45d5-a60c-5301333cbb0c',
             'ext': 'mp4',

From 871645a4a4a0e12ec8f7bf78a3ad7bf75838ee5c Mon Sep 17 00:00:00 2001
From: nixxo <nixxo@protonmail.com>
Date: Sat, 2 Apr 2022 07:57:56 +0200
Subject: [PATCH 493/860] [RAI] Fix extraction of http formats

From https://github.com/yt-dlp/yt-dlp/pull/3272
Closes https://github.com/yt-dlp/yt-dlp/issues/3270
Authored by: nixxo
---
 youtube_dl/extractor/rai.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/rai.py b/youtube_dl/extractor/rai.py
index 7b0315a62..563d3400f 100644
--- a/youtube_dl/extractor/rai.py
+++ b/youtube_dl/extractor/rai.py
@@ -106,7 +106,7 @@ class RaiBaseIE(InfoExtractor):
         }.items() if v is not None)
 
     def _create_http_urls(self, relinker_url, fmts):
-        _RELINKER_REG = r'https?://(?P<host>[^/]+?)/(?:i/)?(?P<extra>[^/]+?)/(?P<path>.+?)/(?P<id>\d+)(?:_(?P<quality>[\d\,]+))?(?:\.mp4|/playlist\.m3u8).+?'
+        _RELINKER_REG = r'https?://(?P<host>[^/]+?)/(?:i/)?(?P<extra>[^/]+?)/(?P<path>.+?)/(?P<id>\w+)(?:_(?P<quality>[\d\,]+))?(?:\.mp4|/playlist\.m3u8).+?'
         _MP4_TMPL = '%s&overrideUserAgentRule=mp4-%s'
         _QUALITY = {
             # tbr: w, h

From b764dbe7730bc5b0a4f30f4f89fd85e096d0c4a0 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 10 Apr 2022 05:49:09 +0100
Subject: [PATCH 494/860] Disable blank issues

---
 .github/ISSUE_TEMPLATE/config.yml | 1 +
 1 file changed, 1 insertion(+)
 create mode 100644 .github/ISSUE_TEMPLATE/config.yml

diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml
new file mode 100644
index 000000000..3ba13e0ce
--- /dev/null
+++ b/.github/ISSUE_TEMPLATE/config.yml
@@ -0,0 +1 @@
+blank_issues_enabled: false

From a0068bd6bec16008bda7a39caecccbf84881c603 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 15 Apr 2022 16:07:09 +0100
Subject: [PATCH 495/860] [Youtube] Fix "n" descrambling for player fae06c11

Resolves #30856.
---
 youtube_dl/extractor/youtube.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 41695a561..ff6c7b0f8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1464,15 +1464,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
     # 2. https://code.videolan.org/videolan/vlc/-/blob/4fb284e5af69aa9ac2100ccbdd3b88debec9987f/share/lua/playlist/youtube.lua#L116
     # 3. https://github.com/ytdl-org/youtube-dl/issues/30097#issuecomment-950157377
     def _extract_n_function_name(self, jscode):
-        target = r'(?P<nfunc>[a-zA-Z0-9$]{3})(?:\[(?P<idx>\d+)\])?'
+        target = r'(?P<nfunc>[a-zA-Z_$][\w$]*)(?:\[(?P<idx>\d+)\])?'
         nfunc_and_idx = self._search_regex(
-            r'\.get\("n"\)\)&&\(b=(%s)\([a-zA-Z0-9]\)' % (target, ),
+            r'\.get\("n"\)\)&&\(b=(%s)\([\w$]+\)' % (target, ),
             jscode, 'Initial JS player n function name')
         nfunc, idx = re.match(target, nfunc_and_idx).group('nfunc', 'idx')
         if not idx:
             return nfunc
         return self._parse_json(self._search_regex(
-            r'var %s\s*=\s*(\[.+?\]);' % (nfunc, ), jscode,
+            r'var %s\s*=\s*(\[.+?\]);' % (re.escape(nfunc), ), jscode,
             'Initial JS player n function list ({nfunc}[{idx}])'.format(**locals())), nfunc, transform_source=js_to_json)[int(idx)]
 
     def _extract_n_function(self, video_id, player_url):

From ebc627847cd1f5faddf4bd90376c1635777283cf Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?=C3=81rni=20Dagur?= <arni@dagur.eu>
Date: Thu, 28 Apr 2022 11:18:10 +0200
Subject: [PATCH 496/860] [KTH] Add new extractor for KTH play (#30885)

* Implement extractor for KTH play
* Make KTH Play url regex more relaxed
---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/kaltura.py    |  2 +-
 youtube_dl/extractor/kth.py        | 31 ++++++++++++++++++++++++++++++
 3 files changed, 33 insertions(+), 1 deletion(-)
 create mode 100644 youtube_dl/extractor/kth.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 535080d0a..452caeade 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -557,6 +557,7 @@ from .kinja import KinjaEmbedIE
 from .kinopoisk import KinoPoiskIE
 from .konserthusetplay import KonserthusetPlayIE
 from .krasview import KrasViewIE
+from .kth import KTHIE
 from .ku6 import Ku6IE
 from .kusi import KUSIIE
 from .kuwo import (
diff --git a/youtube_dl/extractor/kaltura.py b/youtube_dl/extractor/kaltura.py
index c731612c4..6d4d93394 100644
--- a/youtube_dl/extractor/kaltura.py
+++ b/youtube_dl/extractor/kaltura.py
@@ -373,5 +373,5 @@ class KalturaIE(InfoExtractor):
             'duration': info.get('duration'),
             'timestamp': info.get('createdAt'),
             'uploader_id': info.get('userId') if info.get('userId') != 'None' else None,
-            'view_count': info.get('plays'),
+            'view_count': int_or_none(info.get('plays')),
         }
diff --git a/youtube_dl/extractor/kth.py b/youtube_dl/extractor/kth.py
new file mode 100644
index 000000000..b8db461f5
--- /dev/null
+++ b/youtube_dl/extractor/kth.py
@@ -0,0 +1,31 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import smuggle_url
+
+
+class KTHIE(InfoExtractor):
+    _VALID_URL = r'https?://play\.kth\.se/(?:[^/]+/)+(?P<id>[a-z0-9_]+)'
+    _TEST = {
+        'url': 'https://play.kth.se/media/Lunch+breakA+De+nya+aff%C3%A4rerna+inom+Fordonsdalen/0_uoop6oz9',
+        'md5': 'd83ada6d00ca98b73243a88efe19e8a6',
+        'info_dict': {
+            'id': '0_uoop6oz9',
+            'ext': 'mp4',
+            'title': 'md5:bd1d6931facb6828762a33e6ce865f37',
+            'thumbnail': 're:https?://.+/thumbnail/.+',
+            'duration': 3516,
+            'timestamp': 1647345358,
+            'upload_date': '20220315',
+            'uploader_id': 'md5:0ec23e33a89e795a4512930c8102509f',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        result = self.url_result(
+            smuggle_url('kaltura:308:%s' % video_id, {
+                'service_url': 'https://api.kaltura.nordu.net'}),
+            'Kaltura')
+        return result

From e27d8d819fa69d5714ea1682a1d5d56f617461fc Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 29 Apr 2022 13:36:02 +0100
Subject: [PATCH 497/860] [streamcz] Remove empty `'{}'.format()` for Py2.6

Use `'-join()'` here, or `{0}`, ..., in general.
---
 youtube_dl/extractor/streamcz.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/streamcz.py b/youtube_dl/extractor/streamcz.py
index 060ba32e0..97b2eb7f8 100644
--- a/youtube_dl/extractor/streamcz.py
+++ b/youtube_dl/extractor/streamcz.py
@@ -62,7 +62,7 @@ class StreamCZIE(InfoExtractor):
                 if not stream.get('url'):
                     continue
                 yield merge_dicts({
-                    'format_id': '{}-{}'.format(format_id, ext),
+                    'format_id': '-'.join((format_id, ext)),
                     'ext': ext,
                     'source_preference': pref,
                     'url': urljoin(spl_url, stream['url']),

From e988fa4523e489596a2a27c4d45275e44db49406 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 28 Apr 2022 15:25:49 +0100
Subject: [PATCH 498/860] [doc] Clarify test naming

---
 README.md | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 2841ed68f..cd888c731 100644
--- a/README.md
+++ b/README.md
@@ -1069,9 +1069,11 @@ After you have ensured this site is distributing its content legally, you can fo
             }
     ```
 5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
-6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in.
-7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
-8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):
+6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test (actually, test case) then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note:
+    * the test names use the extractor class name **without the trailing `IE`**
+    * tests with `only_matching` key in test's dict are not counted.
+8. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
+9. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):
 
         $ flake8 youtube_dl/extractor/yourextractor.py
 

From c7965b9fc2cae54f244f31f5373cb81a40e822ab Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 9 May 2022 18:54:41 +0100
Subject: [PATCH 499/860] [NHK] Support alphabetic characters in 7-char NhkVod
 IDs (#29682)

---
 youtube_dl/extractor/nhk.py | 19 +++++++++++++++++--
 1 file changed, 17 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nhk.py b/youtube_dl/extractor/nhk.py
index 8a9331a79..46a800e7e 100644
--- a/youtube_dl/extractor/nhk.py
+++ b/youtube_dl/extractor/nhk.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -23,7 +24,7 @@ class NhkBaseIE(InfoExtractor):
     def _extract_episode_info(self, url, episode=None):
         fetch_episode = episode is None
         lang, m_type, episode_id = re.match(NhkVodIE._VALID_URL, url).groups()
-        if episode_id.isdigit():
+        if len(episode_id) == 7:
             episode_id = episode_id[:4] + '-' + episode_id[4:]
 
         is_video = m_type == 'video'
@@ -84,7 +85,8 @@ class NhkBaseIE(InfoExtractor):
 
 
 class NhkVodIE(NhkBaseIE):
-    _VALID_URL = r'%s%s(?P<id>\d{7}|[^/]+?-\d{8}-[0-9a-z]+)' % (NhkBaseIE._BASE_URL_REGEX, NhkBaseIE._TYPE_REGEX)
+    # the 7-character IDs can have alphabetic chars too: assume [a-z] rather than just [a-f], eg
+    _VALID_URL = r'%s%s(?P<id>[0-9a-z]{7}|[^/]+?-\d{8}-[0-9a-z]+)' % (NhkBaseIE._BASE_URL_REGEX, NhkBaseIE._TYPE_REGEX)
     # Content available only for a limited period of time. Visit
     # https://www3.nhk.or.jp/nhkworld/en/ondemand/ for working samples.
     _TESTS = [{
@@ -124,6 +126,19 @@ class NhkVodIE(NhkBaseIE):
     }, {
         'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/audio/j_art-20150903-1/',
         'only_matching': True,
+    }, {
+        # video, alphabetic character in ID #29670
+        'url': 'https://www3.nhk.or.jp/nhkworld/en/ondemand/video/9999a34/',
+        'only_matching': True,
+        'info_dict': {
+            'id': 'qfjay6cg',
+            'ext': 'mp4',
+            'title': 'DESIGN TALKS plus - Fishermen’s Finery',
+            'description': 'md5:8a8f958aaafb0d7cb59d38de53f1e448',
+            'thumbnail': r're:^https?:/(/[a-z0-9.-]+)+\.jpg\?w=1920&h=1080$',
+            'upload_date': '20210615',
+            'timestamp': 1623722008,
+        }
     }]
 
     def _real_extract(self, url):

From c3deca86aedd2d8ab7cd0c596fd68b7aeb7c042d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 19 May 2022 17:41:48 +0000
Subject: [PATCH 500/860] [wat.tv] Add version `pver` to metadata API call

Resolves #30959.
---
 youtube_dl/extractor/wat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/wat.py b/youtube_dl/extractor/wat.py
index f1bccc2d6..b15e03768 100644
--- a/youtube_dl/extractor/wat.py
+++ b/youtube_dl/extractor/wat.py
@@ -57,7 +57,7 @@ class WatIE(InfoExtractor):
         #     'http://www.wat.tv/interface/contentv4s/' + video_id, video_id)
         video_data = self._download_json(
             'https://mediainfo.tf1.fr/mediainfocombo/' + video_id,
-            video_id, query={'context': 'MYTF1'})
+            video_id, query={'context': 'MYTF1', 'pver': '4001000'})
         video_info = video_data['media']
 
         error_desc = video_info.get('error_desc')

From be35e5343a6c31f5f32ee216ab4486a1992260c5 Mon Sep 17 00:00:00 2001
From: Jacob Chapman <7908073+chapmanjacobd@users.noreply.github.com>
Date: Wed, 13 Apr 2022 07:21:23 -0500
Subject: [PATCH 501/860] Update options.py

---
 youtube_dl/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 0a0641bd4..6521ad881 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -270,11 +270,11 @@ def parseOpts(overrideArguments=None):
     selection.add_option(
         '--match-title',
         dest='matchtitle', metavar='REGEX',
-        help='Download only matching titles (regex or caseless sub-string)')
+        help='Download only matching titles (case-insensitive regex or sub-string)')
     selection.add_option(
         '--reject-title',
         dest='rejecttitle', metavar='REGEX',
-        help='Skip download for matching titles (regex or caseless sub-string)')
+        help='Skip download for matching titles (case-insensitive regex or sub-string)')
     selection.add_option(
         '--max-downloads',
         dest='max_downloads', metavar='NUMBER', type=int, default=None,

From 187a48aee29847664e0c4cd80fe90c32e1fb334b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 24 May 2022 15:33:00 +0100
Subject: [PATCH 502/860] [YouTube] Handle player c5a4daa1 with indirect
 n-function definition

* resolves #30976
---
 youtube_dl/extractor/youtube.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ff6c7b0f8..9c62b8890 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1471,9 +1471,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         nfunc, idx = re.match(target, nfunc_and_idx).group('nfunc', 'idx')
         if not idx:
             return nfunc
+        if int_or_none(idx) == 0:
+            real_nfunc = self._search_regex(
+                r'var %s\s*=\s*\[([a-zA-Z_$][\w$]*)\];' % (re.escape(nfunc), ), jscode,
+                'Initial JS player n function alias ({nfunc}[{idx}])'.format(**locals()))
+            if real_nfunc:
+                return real_nfunc
         return self._parse_json(self._search_regex(
             r'var %s\s*=\s*(\[.+?\]);' % (re.escape(nfunc), ), jscode,
-            'Initial JS player n function list ({nfunc}[{idx}])'.format(**locals())), nfunc, transform_source=js_to_json)[int(idx)]
+            'Initial JS player n function name ({nfunc}[{idx}])'.format(**locals())), nfunc, transform_source=js_to_json)[int(idx)]
 
     def _extract_n_function(self, video_id, player_url):
         player_id = self._extract_player_info(player_url)
@@ -1482,7 +1488,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if func_code:
             jsi = JSInterpreter(func_code)
         else:
-            player_id = self._extract_player_info(player_url)
             jscode = self._get_player_code(video_id, player_url, player_id)
             funcname = self._extract_n_function_name(jscode)
             jsi = JSInterpreter(jscode)

From 52c3751df722ab6f31f0229a415c7389a95c2307 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 28 May 2022 13:52:51 +0100
Subject: [PATCH 503/860] [utils] Enable ALPN in HTTPS to satisfy broken
 servers

See https://github.com/yt-dlp/yt-dlp/issues/3878
---
 youtube_dl/utils.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e722eed58..4ff27db3d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2292,12 +2292,30 @@ def formatSeconds(secs):
 
 
 def make_HTTPS_handler(params, **kwargs):
+
+    # https://www.rfc-editor.org/info/rfc7301
+    ALPN_PROTOCOLS = ['http/1.1']
+
+    def set_alpn_protocols(ctx):
+        # From https://github.com/yt-dlp/yt-dlp/commit/2c6dcb65fb612fc5bc5c61937bf438d3c473d8d0
+        # Thanks @coletdjnz
+        # Some servers may (wrongly) reject requests if ALPN extension is not sent. See:
+        # https://github.com/python/cpython/issues/85140
+        # https://github.com/yt-dlp/yt-dlp/issues/3878
+        try:
+            ctx.set_alpn_protocols(ALPN_PROTOCOLS)
+        except (AttributeError, NotImplementedError):
+            # Python < 2.7.10, not ssl.HAS_ALPN
+            pass
+
     opts_no_check_certificate = params.get('nocheckcertificate', False)
     if hasattr(ssl, 'create_default_context'):  # Python >= 3.4 or 2.7.9
         context = ssl.create_default_context(ssl.Purpose.SERVER_AUTH)
+        set_alpn_protocols(context)
         if opts_no_check_certificate:
             context.check_hostname = False
             context.verify_mode = ssl.CERT_NONE
+
         try:
             return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
         except TypeError:
@@ -2313,6 +2331,7 @@ def make_HTTPS_handler(params, **kwargs):
                                if opts_no_check_certificate
                                else ssl.CERT_REQUIRED)
         context.set_default_verify_paths()
+        set_alpn_protocols(context)
         return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
 
 

From 04fd3289d30de3c99c7d2de34d555b050bc96d4d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 28 May 2022 13:54:32 +0100
Subject: [PATCH 504/860] [YouPorn] Improve `upload_date` extraction

See https://github.com/yt-dlp/yt-dlp/issues/2701#issuecomment-1034341883
---
 youtube_dl/extractor/youporn.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 7084d3d12..31e8abb72 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -137,9 +138,10 @@ class YouPornIE(InfoExtractor):
             r'(?s)<div[^>]+class=["\']submitByLink["\'][^>]*>(.+?)</div>',
             webpage, 'uploader', fatal=False)
         upload_date = unified_strdate(self._html_search_regex(
-            [r'UPLOADED:\s*<span>([^<]+)',
+            (r'UPLOADED:\s*<span>([^<]+)',
              r'Date\s+[Aa]dded:\s*<span>([^<]+)',
-             r'(?s)<div[^>]+class=["\']videoInfo(?:Date|Time)["\'][^>]*>(.+?)</div>'],
+             r'''(?s)<div[^>]+class=["']videoInfo(?:Date|Time)\b[^>]*>(.+?)</div>''',
+             r'(?s)<label\b[^>]*>Uploaded[^<]*</label>\s*<span\b[^>]*>(.+?)</span>'),
             webpage, 'upload date', fatal=False))
 
         age_limit = self._rta_search(webpage)

From 9aa8e5340f3d5ece372b983f8e399277ca1f1fe4 Mon Sep 17 00:00:00 2001
From: LewdyCoder <88900506+LewdyCoder@users.noreply.github.com>
Date: Mon, 30 May 2022 03:50:50 +0200
Subject: [PATCH 505/860] [Readme] Clarified extractor naming (#29799)

* Exported usable extractors must be named `xxxxIE`

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 CONTRIBUTING.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md
index 58ab3a4b8..ff40cef78 100644
--- a/CONTRIBUTING.md
+++ b/CONTRIBUTING.md
@@ -150,7 +150,7 @@ After you have ensured this site is distributing its content legally, you can fo
                 # TODO more properties (see youtube_dl/extractor/common.py)
             }
     ```
-5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py).
+5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py). This makes the extractor available for use, as long as the class ends with `IE`.
 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in.
 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want.
 8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart):

From 1baa0f5f6678c047624785dc9a3ab3cb44a72809 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 29 Apr 2021 04:56:09 +0530
Subject: [PATCH 506/860] [utils] Escape URL while sanitizing Closes #31008,
 #yt-dlp/263

While this fixes the issue in question, it does not try to address the root-cause of the problem
Refer: 915f911e365736227e134ad654601443dbfd7ccb, f5fa042c82300218a2d07b95dd6b9c0756745db3
---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4ff27db3d..8aa2a43a2 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2151,7 +2151,7 @@ def sanitize_url(url):
     for mistake, fixup in COMMON_TYPOS:
         if re.match(mistake, url):
             return re.sub(mistake, fixup, url)
-    return url
+    return escape_url(url)
 
 
 def sanitized_Request(url, *args, **kwargs):

From 530f4582d011cd94986cf4d233f9fb9263f72150 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 6 Jun 2022 19:29:48 +0100
Subject: [PATCH 507/860] [HRFernsehen] Back-port new extractor from yt-dlp
 Closes #26445, where this was originally proposed.

---
 youtube_dl/extractor/extractors.py  |   1 +
 youtube_dl/extractor/hrfernsehen.py | 101 ++++++++++++++++++++++++++++
 2 files changed, 102 insertions(+)
 create mode 100644 youtube_dl/extractor/hrfernsehen.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 452caeade..751fc38b6 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -479,6 +479,7 @@ from .hotstar import (
 )
 from .howcast import HowcastIE
 from .howstuffworks import HowStuffWorksIE
+from .hrfernsehen import HRFernsehenIE
 from .hrti import (
     HRTiIE,
     HRTiPlaylistIE,
diff --git a/youtube_dl/extractor/hrfernsehen.py b/youtube_dl/extractor/hrfernsehen.py
new file mode 100644
index 000000000..11b879dbd
--- /dev/null
+++ b/youtube_dl/extractor/hrfernsehen.py
@@ -0,0 +1,101 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+import json
+import re
+
+from ..utils import (
+    int_or_none,
+    unified_timestamp,
+    unescapeHTML
+)
+from .common import InfoExtractor
+
+
+class HRFernsehenIE(InfoExtractor):
+    IE_NAME = 'hrfernsehen'
+    _VALID_URL = r'^https?://www\.(?:hr-fernsehen|hessenschau)\.de/.*,video-(?P<id>[0-9]{6})\.html'
+
+    _TESTS = [{
+        'url': 'https://www.hessenschau.de/tv-sendung/hessenschau-vom-26082020,video-130546.html',
+        'md5': '5c4e0ba94677c516a2f65a84110fc536',
+        'info_dict': {
+            'id': '130546',
+            'ext': 'mp4',
+            'description': 'Sturmtief Kirsten fegt über Hessen / Die Corona-Pandemie – eine Chronologie / '
+                           'Sterbehilfe: Die Lage in Hessen / Miss Hessen leitet zwei eigene Unternehmen / '
+                           'Pop-Up Museum zeigt Schwarze Unterhaltung und Black Music',
+            'subtitles': {'de': [{
+                'url': 'https://hr-a.akamaihd.net/video/as/hessenschau/2020_08/hrLogo_200826200407_L385592_512x288-25p-500kbit.vtt'
+            }]},
+            'timestamp': 1598470200,
+            'upload_date': '20200826',
+            'thumbnail': 'https://www.hessenschau.de/tv-sendung/hs_ganz-1554~_t-1598465545029_v-16to9__medium.jpg',
+            'title': 'hessenschau vom 26.08.2020'
+        }
+    }, {
+        'url': 'https://www.hr-fernsehen.de/sendungen-a-z/mex/sendungen/fair-und-gut---was-hinter-aldis-eigenem-guetesiegel-steckt,video-130544.html',
+        'only_matching': True
+    }]
+
+    _GEO_COUNTRIES = ['DE']
+
+    def extract_airdate(self, loader_data):
+        airdate_str = loader_data.get('mediaMetadata', {}).get('agf', {}).get('airdate')
+
+        if airdate_str is None:
+            return None
+
+        return unified_timestamp(airdate_str)
+
+    def extract_formats(self, loader_data):
+        stream_formats = []
+        for stream_obj in loader_data["videoResolutionLevels"]:
+            stream_format = {
+                'format_id': str(stream_obj['verticalResolution']) + "p",
+                'height': stream_obj['verticalResolution'],
+                'url': stream_obj['url'],
+            }
+
+            quality_information = re.search(r'([0-9]{3,4})x([0-9]{3,4})-([0-9]{2})p-([0-9]{3,4})kbit',
+                                            stream_obj['url'])
+            if quality_information:
+                stream_format['width'] = int_or_none(quality_information.group(1))
+                stream_format['height'] = int_or_none(quality_information.group(2))
+                stream_format['fps'] = int_or_none(quality_information.group(3))
+                stream_format['tbr'] = int_or_none(quality_information.group(4))
+
+            stream_formats.append(stream_format)
+
+        self._sort_formats(stream_formats)
+        return stream_formats
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = self._html_search_meta(
+            ['og:title', 'twitter:title', 'name'], webpage)
+        description = self._html_search_meta(
+            ['description'], webpage)
+
+        loader_str = unescapeHTML(self._search_regex(r"data-new-hr-mediaplayer-loader='([^']*)'", webpage, "ardloader"))
+        loader_data = json.loads(loader_str)
+
+        info = {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'formats': self.extract_formats(loader_data),
+            'timestamp': self.extract_airdate(loader_data)
+        }
+
+        if "subtitle" in loader_data:
+            info["subtitles"] = {"de": [{"url": loader_data["subtitle"]}]}
+
+        thumbnails = list(set([t for t in loader_data.get("previewImageUrl", {}).values()]))
+        if len(thumbnails) > 0:
+            info["thumbnails"] = [{"url": t} for t in thumbnails]
+
+        return info

From ef044be34bb64c489558dd07818616b514d2e2ad Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 8 Jun 2022 15:52:21 +0100
Subject: [PATCH 508/860] [test] Skip not _WORKING IE in subtitle tests; use
 unittest.skipTest throughout

---
 test/test_download.py  | 7 +++----
 test/test_subtitles.py | 3 +++
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/test/test_download.py b/test/test_download.py
index 8e43cfa12..0951a171a 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -88,7 +88,6 @@ class TestDownload(unittest.TestCase):
 
 # Dynamically generate tests
 
-
 def generator(test_case, tname):
 
     def test_template(self):
@@ -100,9 +99,10 @@ def generator(test_case, tname):
 
         def print_skipping(reason):
             print('Skipping %s: %s' % (test_case['name'], reason))
+            self.skipTest(reason)
+
         if not ie.working():
             print_skipping('IE marked as not _WORKING')
-            return
 
         for tc in test_cases:
             info_dict = tc.get('info_dict', {})
@@ -111,11 +111,10 @@ def generator(test_case, tname):
 
         if 'skip' in test_case:
             print_skipping(test_case['skip'])
-            return
+
         for other_ie in other_ies:
             if not other_ie.working():
                 print_skipping('test depends on %sIE, marked as not WORKING' % other_ie.ie_key())
-                return
 
         params = get_params(test_case.get('params', {}))
         params['outtmpl'] = tname + '_' + params['outtmpl']
diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 550e0ca00..c250473be 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -38,6 +38,9 @@ class BaseTestSubtitles(unittest.TestCase):
         self.DL = FakeYDL()
         self.ie = self.IE()
         self.DL.add_info_extractor(self.ie)
+        if not self.IE.working():
+            print('Skipping: %s marked as not _WORKING' % self.IE.ie_key())
+            self.skipTest('IE marked as not _WORKING')
 
     def getInfoDict(self):
         info_dict = self.DL.extract_info(self.url, download=False)

From 3aa94d7945dfaa0e04acf2700ffe0e43b00db498 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 8 Jun 2022 23:11:33 +0100
Subject: [PATCH 509/860] [test] Fix workable subtitle tests (except YT) and
 mark others as skip, broken

* broken tests need to be fixed when fixing the respective IE
---
 test/test_subtitles.py | 22 +++++++++++++++-------
 1 file changed, 15 insertions(+), 7 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index c250473be..23cf06e09 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -131,6 +131,7 @@ class TestDailymotionSubtitles(BaseTestSubtitles):
         self.assertFalse(subtitles)
 
 
+@unittest.skip('IE broken')
 class TestTedSubtitles(BaseTestSubtitles):
     url = 'http://www.ted.com/talks/dan_dennett_on_our_consciousness.html'
     IE = TEDIE
@@ -155,18 +156,19 @@ class TestVimeoSubtitles(BaseTestSubtitles):
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['de', 'en', 'es', 'fr']))
-        self.assertEqual(md5(subtitles['en']), '8062383cf4dec168fc40a088aa6d5888')
-        self.assertEqual(md5(subtitles['fr']), 'b6191146a6c5d3a452244d853fde6dc8')
+        self.assertEqual(md5(subtitles['en']), '386cbc9320b94e25cb364b97935e5dd1')
+        self.assertEqual(md5(subtitles['fr']), 'c9b69eef35bc6641c0d4da8a04f9dfac')
 
     def test_nosubtitles(self):
         self.DL.expect_warning('video doesn\'t have subtitles')
-        self.url = 'http://vimeo.com/56015672'
+        self.url = 'http://vimeo.com/68093876'
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertFalse(subtitles)
 
 
+@unittest.skip('IE broken')
 class TestWallaSubtitles(BaseTestSubtitles):
     url = 'http://vod.walla.co.il/movie/2705958/the-yes-men'
     IE = WallaIE
@@ -188,6 +190,7 @@ class TestWallaSubtitles(BaseTestSubtitles):
         self.assertFalse(subtitles)
 
 
+@unittest.skip('IE broken')
 class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
     url = 'http://www.ceskatelevize.cz/ivysilani/10600540290-u6-uzasny-svet-techniky'
     IE = CeskaTelevizeIE
@@ -209,6 +212,7 @@ class TestCeskaTelevizeSubtitles(BaseTestSubtitles):
         self.assertFalse(subtitles)
 
 
+@unittest.skip('IE broken')
 class TestLyndaSubtitles(BaseTestSubtitles):
     url = 'http://www.lynda.com/Bootstrap-tutorials/Using-exercise-files/110885/114408-4.html'
     IE = LyndaIE
@@ -221,6 +225,7 @@ class TestLyndaSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['en']), '09bbe67222259bed60deaa26997d73a7')
 
 
+@unittest.skip('IE broken')
 class TestNPOSubtitles(BaseTestSubtitles):
     url = 'http://www.npo.nl/nos-journaal/28-08-2014/POW_00722860'
     IE = NPOIE
@@ -233,6 +238,7 @@ class TestNPOSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['nl']), 'fc6435027572b63fb4ab143abd5ad3f4')
 
 
+@unittest.skip('IE broken')
 class TestMTVSubtitles(BaseTestSubtitles):
     url = 'http://www.cc.com/video-clips/p63lk0/adam-devine-s-house-party-chasing-white-swans'
     IE = ComedyCentralIE
@@ -256,8 +262,8 @@ class TestNRKSubtitles(BaseTestSubtitles):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
-        self.assertEqual(set(subtitles.keys()), set(['no']))
-        self.assertEqual(md5(subtitles['no']), '544fa917d3197fcbee64634559221cc2')
+        self.assertEqual(set(subtitles.keys()), set(['nb-ttv']))
+        self.assertEqual(md5(subtitles['nb-ttv']), '67e06ff02d0deaf975e68f6cb8f6a149')
 
 
 class TestRaiPlaySubtitles(BaseTestSubtitles):
@@ -280,6 +286,7 @@ class TestRaiPlaySubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['it']), '4b3264186fbb103508abe5311cfcb9cd')
 
 
+@unittest.skip('IE broken - DRM only')
 class TestVikiSubtitles(BaseTestSubtitles):
     url = 'http://www.viki.com/videos/1060846v-punch-episode-18'
     IE = VikiIE
@@ -306,6 +313,7 @@ class TestThePlatformSubtitles(BaseTestSubtitles):
         self.assertEqual(md5(subtitles['en']), '97e7670cbae3c4d26ae8bcc7fdd78d4b')
 
 
+@unittest.skip('IE broken')
 class TestThePlatformFeedSubtitles(BaseTestSubtitles):
     url = 'http://feed.theplatform.com/f/7wvmTC/msnbc_video-p-test?form=json&pretty=true&range=-40&byGuid=n_hardball_5biden_140207'
     IE = ThePlatformFeedIE
@@ -341,7 +349,7 @@ class TestDemocracynowSubtitles(BaseTestSubtitles):
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['en']))
-        self.assertEqual(md5(subtitles['en']), 'acaca989e24a9e45a6719c9b3d60815c')
+        self.assertEqual(md5(subtitles['en']), 'a3cc4c0b5eadd74d9974f1c1f5101045')
 
     def test_subtitles_in_page(self):
         self.url = 'http://www.democracynow.org/2015/7/3/this_flag_comes_down_today_bree'
@@ -349,7 +357,7 @@ class TestDemocracynowSubtitles(BaseTestSubtitles):
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['en']))
-        self.assertEqual(md5(subtitles['en']), 'acaca989e24a9e45a6719c9b3d60815c')
+        self.assertEqual(md5(subtitles['en']), 'a3cc4c0b5eadd74d9974f1c1f5101045')
 
 
 if __name__ == '__main__':

From 811c480f7b6c25ca510a033e6365d00174135392 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 9 Jun 2022 15:25:23 +0100
Subject: [PATCH 510/860] [YouTube] Support JSON3 subtitle format * subtitle
 tests updated to match

---
 test/test_subtitles.py          | 74 ++++++++++++++++++++++++---------
 youtube_dl/extractor/youtube.py |  2 +-
 2 files changed, 55 insertions(+), 21 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 23cf06e09..4cbc69ccd 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -59,6 +59,21 @@ class BaseTestSubtitles(unittest.TestCase):
 
 
 class TestYoutubeSubtitles(BaseTestSubtitles):
+    # Available subtitles for QRS8MkLhQmM:
+    # Language formats
+    # ru       vtt, ttml, srv3, srv2, srv1, json3
+    # fr       vtt, ttml, srv3, srv2, srv1, json3
+    # en       vtt, ttml, srv3, srv2, srv1, json3
+    # nl       vtt, ttml, srv3, srv2, srv1, json3
+    # de       vtt, ttml, srv3, srv2, srv1, json3
+    # ko       vtt, ttml, srv3, srv2, srv1, json3
+    # it       vtt, ttml, srv3, srv2, srv1, json3
+    # zh-Hant  vtt, ttml, srv3, srv2, srv1, json3
+    # hi       vtt, ttml, srv3, srv2, srv1, json3
+    # pt-BR    vtt, ttml, srv3, srv2, srv1, json3
+    # es-MX    vtt, ttml, srv3, srv2, srv1, json3
+    # ja       vtt, ttml, srv3, srv2, srv1, json3
+    # pl       vtt, ttml, srv3, srv2, srv1, json3
     url = 'QRS8MkLhQmM'
     IE = YoutubeIE
 
@@ -67,41 +82,60 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
         self.assertEqual(len(subtitles.keys()), 13)
-        self.assertEqual(md5(subtitles['en']), '3cb210999d3e021bd6c7f0ea751eab06')
-        self.assertEqual(md5(subtitles['it']), '6d752b98c31f1cf8d597050c7a2cb4b5')
+        self.assertEqual(md5(subtitles['en']), 'ae1bd34126571a77aabd4d276b28044d')
+        self.assertEqual(md5(subtitles['it']), '0e0b667ba68411d88fd1c5f4f4eab2f9')
         for lang in ['fr', 'de']:
             self.assertTrue(subtitles.get(lang) is not None, 'Subtitles for \'%s\' not extracted' % lang)
 
-    def test_youtube_subtitles_ttml_format(self):
+    def _test_subtitles_format(self, fmt, md5_hash, lang='en'):
         self.DL.params['writesubtitles'] = True
-        self.DL.params['subtitlesformat'] = 'ttml'
+        self.DL.params['subtitlesformat'] = fmt
         subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), 'e306f8c42842f723447d9f63ad65df54')
+        self.assertEqual(md5(subtitles[lang]), md5_hash)
+
+    def test_youtube_subtitles_ttml_format(self):
+        self._test_subtitles_format('ttml', 'c97ddf1217390906fa9fbd34901f3da2')
 
     def test_youtube_subtitles_vtt_format(self):
-        self.DL.params['writesubtitles'] = True
-        self.DL.params['subtitlesformat'] = 'vtt'
+        self._test_subtitles_format('vtt', 'ae1bd34126571a77aabd4d276b28044d')
+
+    def test_youtube_subtitles_json3_format(self):
+        self._test_subtitles_format('json3', '688dd1ce0981683867e7fe6fde2a224b')
+
+    def _test_automatic_captions(self, url, lang):
+        self.url = url
+        self.DL.params['writeautomaticsub'] = True
+        self.DL.params['subtitleslangs'] = [lang]
         subtitles = self.getSubtitles()
-        self.assertEqual(md5(subtitles['en']), '3cb210999d3e021bd6c7f0ea751eab06')
+        self.assertTrue(subtitles[lang] is not None)
 
     def test_youtube_automatic_captions(self):
-        self.url = '8YoUxe5ncPo'
-        self.DL.params['writeautomaticsub'] = True
-        self.DL.params['subtitleslangs'] = ['it']
-        subtitles = self.getSubtitles()
-        self.assertTrue(subtitles['it'] is not None)
+        # Available automatic captions for 8YoUxe5ncPo:
+        # Language formats (all in vtt, ttml, srv3, srv2, srv1, json3)
+        # gu, zh-Hans, zh-Hant, gd, ga, gl, lb, la, lo, tt, tr,
+        # lv, lt, tk, th, tg, te, fil, haw, yi, ceb, yo, de, da,
+        # el, eo, en, eu, et, es, ru, rw, ro, bn, be, bg, uk, jv,
+        # bs, ja, or, xh, co, ca, cy, cs, ps, pt, pa, vi, pl, hy,
+        # hr, ht, hu, hmn, hi, ha, mg, uz, ml, mn, mi, mk, ur,
+        # mt, ms, mr, ug, ta, my, af, sw, is, am, 
+        #                                         *it*, iw, sv, ar,
+        # su, zu, az, id, ig, nl, no, ne, ny, fr, ku, fy, fa, fi,
+        # ka, kk, sr, sq, ko, kn, km, st, sk, si, so, sn, sm, sl,
+        # ky, sd
+        # ...
+        self._test_automatic_captions('8YoUxe5ncPo', 'it')
 
+    @unittest.skip('ASR subs all in all supported langs now')
     def test_youtube_translated_subtitles(self):
-        # This video has a subtitles track, which can be translated
-        self.url = 'Ky9eprVWzlI'
-        self.DL.params['writeautomaticsub'] = True
-        self.DL.params['subtitleslangs'] = ['it']
-        subtitles = self.getSubtitles()
-        self.assertTrue(subtitles['it'] is not None)
+        # This video has a subtitles track, which can be translated (#4555)
+        self._test_automatic_captions('Ky9eprVWzlI', 'it')
 
     def test_youtube_nosubtitles(self):
         self.DL.expect_warning('video doesn\'t have subtitles')
-        self.url = 'n5BB19UTcdA'
+        # Available automatic captions for 8YoUxe5ncPo:
+        # ...
+        # 8YoUxe5ncPo has no subtitles
+        self.url = '8YoUxe5ncPo'
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
         subtitles = self.getSubtitles()
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9c62b8890..91a3b6058 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -499,7 +499,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
     )
-    _SUBTITLE_FORMATS = ('srv1', 'srv2', 'srv3', 'ttml', 'vtt')
+    _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
 
     _GEO_BYPASS = False
 

From 0700fde6403aa9eec1ff02bff7323696a205900c Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan@gmail.com>
Date: Sat, 9 Jan 2021 17:56:12 +0530
Subject: [PATCH 511/860] [utils, etc] Kill child processes when yt-dl is
 killed

* derived from PR #26592, closes #26592

Authored by: Unrud
---
 youtube_dl/YoutubeDL.py                    |  3 ++-
 youtube_dl/compat.py                       |  3 ++-
 youtube_dl/downloader/external.py          | 16 ++++++++++------
 youtube_dl/downloader/rtmp.py              | 10 ++++++----
 youtube_dl/extractor/openload.py           |  3 ++-
 youtube_dl/postprocessor/embedthumbnail.py |  5 +++--
 youtube_dl/postprocessor/ffmpeg.py         |  5 +++--
 youtube_dl/utils.py                        | 18 ++++++++++++++----
 8 files changed, 42 insertions(+), 21 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 019e309cb..3895b408f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -73,6 +73,7 @@ from .utils import (
     PostProcessingError,
     preferredencoding,
     prepend_extension,
+    process_communicate_or_kill,
     register_socks_protocols,
     render_table,
     replace_extension,
@@ -2323,7 +2324,7 @@ class YoutubeDL(object):
                 ['git', 'rev-parse', '--short', 'HEAD'],
                 stdout=subprocess.PIPE, stderr=subprocess.PIPE,
                 cwd=os.path.dirname(os.path.abspath(__file__)))
-            out, err = sp.communicate()
+            out, err = process_communicate_or_kill(sp)
             out = out.decode().strip()
             if re.match('[0-9a-f]+', out):
                 self._write_string('[debug] Git HEAD: ' + out + '\n')
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 2004a405a..9f5f85dae 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2890,6 +2890,7 @@ else:
     _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
 
     def compat_get_terminal_size(fallback=(80, 24)):
+        from .utils import process_communicate_or_kill
         columns = compat_getenv('COLUMNS')
         if columns:
             columns = int(columns)
@@ -2906,7 +2907,7 @@ else:
                 sp = subprocess.Popen(
                     ['stty', 'size'],
                     stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-                out, err = sp.communicate()
+                out, err = process_communicate_or_kill(sp)
                 _lines, _columns = map(int, out.split())
             except Exception:
                 _columns, _lines = _terminal_size(*fallback)
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index c31f8910a..a06ab2e50 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -22,6 +22,7 @@ from ..utils import (
     handle_youtubedl_headers,
     check_executable,
     is_outdated_version,
+    process_communicate_or_kill,
 )
 
 
@@ -104,7 +105,7 @@ class ExternalFD(FileDownloader):
 
         p = subprocess.Popen(
             cmd, stderr=subprocess.PIPE)
-        _, stderr = p.communicate()
+        _, stderr = process_communicate_or_kill(p)
         if p.returncode != 0:
             self.to_stderr(stderr.decode('utf-8', 'replace'))
         return p.returncode
@@ -141,7 +142,7 @@ class CurlFD(ExternalFD):
 
         # curl writes the progress to stderr so don't capture it.
         p = subprocess.Popen(cmd)
-        p.communicate()
+        process_communicate_or_kill(p)
         return p.returncode
 
 
@@ -336,14 +337,17 @@ class FFmpegFD(ExternalFD):
         proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env)
         try:
             retval = proc.wait()
-        except KeyboardInterrupt:
-            # subprocces.run would send the SIGKILL signal to ffmpeg and the
+        except BaseException as e:
+            # subprocess.run would send the SIGKILL signal to ffmpeg and the
             # mp4 file couldn't be played, but if we ask ffmpeg to quit it
             # produces a file that is playable (this is mostly useful for live
             # streams). Note that Windows is not affected and produces playable
             # files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
-            if sys.platform != 'win32':
-                proc.communicate(b'q')
+            if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32':
+                process_communicate_or_kill(proc, b'q')
+            else:
+                proc.kill()
+                proc.wait()
             raise
         return retval
 
diff --git a/youtube_dl/downloader/rtmp.py b/youtube_dl/downloader/rtmp.py
index fbb7f51b0..8a25dbc8d 100644
--- a/youtube_dl/downloader/rtmp.py
+++ b/youtube_dl/downloader/rtmp.py
@@ -89,11 +89,13 @@ class RtmpFD(FileDownloader):
                                 self.to_screen('')
                             cursor_in_new_line = True
                             self.to_screen('[rtmpdump] ' + line)
-            finally:
+                if not cursor_in_new_line:
+                    self.to_screen('')
+                return proc.wait()
+            except BaseException:  # Including KeyboardInterrupt
+                proc.kill()
                 proc.wait()
-            if not cursor_in_new_line:
-                self.to_screen('')
-            return proc.returncode
+                raise
 
         url = info_dict['url']
         player_url = info_dict.get('player_url')
diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index 0c20d0177..b05d60435 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -16,6 +16,7 @@ from ..utils import (
     ExtractorError,
     get_exe_version,
     is_outdated_version,
+    process_communicate_or_kill,
     std_headers,
 )
 
@@ -226,7 +227,7 @@ class PhantomJSwrapper(object):
             self.exe, '--ssl-protocol=any',
             self._TMP_FILES['script'].name
         ], stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-        out, err = p.communicate()
+        out, err = process_communicate_or_kill(p)
         if p.returncode != 0:
             raise ExtractorError(
                 'Executing JS failed\n:' + encodeArgument(err))
diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 3990908b6..5e7b6e2df 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -13,8 +13,9 @@ from ..utils import (
     encodeFilename,
     PostProcessingError,
     prepend_extension,
+    process_communicate_or_kill,
     replace_extension,
-    shell_quote
+    shell_quote,
 )
 
 
@@ -109,7 +110,7 @@ class EmbedThumbnailPP(FFmpegPostProcessor):
                 self._downloader.to_screen('[debug] AtomicParsley command line: %s' % shell_quote(cmd))
 
             p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-            stdout, stderr = p.communicate()
+            stdout, stderr = process_communicate_or_kill(p)
 
             if p.returncode != 0:
                 msg = stderr.decode('utf-8', 'replace').strip()
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 9f76c9d4e..8c29c8d59 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -16,6 +16,7 @@ from ..utils import (
     is_outdated_version,
     PostProcessingError,
     prepend_extension,
+    process_communicate_or_kill,
     shell_quote,
     subtitles_filename,
     dfxp2srt,
@@ -180,7 +181,7 @@ class FFmpegPostProcessor(PostProcessor):
             handle = subprocess.Popen(
                 cmd, stderr=subprocess.PIPE,
                 stdout=subprocess.PIPE, stdin=subprocess.PIPE)
-            stdout_data, stderr_data = handle.communicate()
+            stdout_data, stderr_data = process_communicate_or_kill(handle)
             expected_ret = 0 if self.probe_available else 1
             if handle.wait() != expected_ret:
                 return None
@@ -228,7 +229,7 @@ class FFmpegPostProcessor(PostProcessor):
         if self._downloader.params.get('verbose', False):
             self._downloader.to_screen('[debug] ffmpeg command line: %s' % shell_quote(cmd))
         p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
-        stdout, stderr = p.communicate()
+        stdout, stderr = process_communicate_or_kill(p)
         if p.returncode != 0:
             stderr = stderr.decode('utf-8', 'replace')
             msgs = stderr.strip().split('\n')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 8aa2a43a2..4e00317f1 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2212,6 +2212,15 @@ def unescapeHTML(s):
         r'&([^&;]+;)', lambda m: _htmlentity_transform(m.group(1)), s)
 
 
+def process_communicate_or_kill(p, *args, **kwargs):
+    try:
+        return p.communicate(*args, **kwargs)
+    except BaseException:  # Including KeyboardInterrupt
+        p.kill()
+        p.wait()
+        raise
+
+
 def get_subprocess_encoding():
     if sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
         # For subprocess calls, encode with locale encoding
@@ -3788,7 +3797,8 @@ def check_executable(exe, args=[]):
     """ Checks if the given binary is installed somewhere in PATH, and returns its name.
     args can be a list of arguments for a short output (like -version) """
     try:
-        subprocess.Popen([exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE).communicate()
+        process_communicate_or_kill(subprocess.Popen(
+            [exe] + args, stdout=subprocess.PIPE, stderr=subprocess.PIPE))
     except OSError:
         return False
     return exe
@@ -3802,10 +3812,10 @@ def get_exe_version(exe, args=['--version'],
         # STDIN should be redirected too. On UNIX-like systems, ffmpeg triggers
         # SIGTTOU if youtube-dl is run in the background.
         # See https://github.com/ytdl-org/youtube-dl/issues/955#issuecomment-209789656
-        out, _ = subprocess.Popen(
+        out, _ = process_communicate_or_kill(subprocess.Popen(
             [encodeArgument(exe)] + args,
             stdin=subprocess.PIPE,
-            stdout=subprocess.PIPE, stderr=subprocess.STDOUT).communicate()
+            stdout=subprocess.PIPE, stderr=subprocess.STDOUT))
     except OSError:
         return False
     if isinstance(out, bytes):  # Python 2.x
@@ -5744,7 +5754,7 @@ def write_xattr(path, key, value):
                         cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, stdin=subprocess.PIPE)
                 except EnvironmentError as e:
                     raise XAttrMetadataError(e.errno, e.strerror)
-                stdout, stderr = p.communicate()
+                stdout, stderr = process_communicate_or_kill(p)
                 stderr = stderr.decode('utf-8', 'replace')
                 if p.returncode != 0:
                     raise XAttrMetadataError(p.returncode, stderr)

From cc179df346abf34c8f77dbb221b839092007f20c Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 12 Jun 2022 14:10:38 +0100
Subject: [PATCH 512/860] [XHamster] Support xhday.com alias, extract
 `uploader_id` * support xhday.com alias for xhamster.com (resolves #31023)  
 Authored by: dirkf * extract `uploader_id`:   from
 https://github.com/yt-dlp/yt-dlp/commit/908b56eaf7872149706dbd7fa071f838d0c786b7
   (PR https://github.com/yt-dlp/yt-dlp/pull/844)   Authored by: octotherp

---
 youtube_dl/extractor/xhamster.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index f73b9778f..f764021ba 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import itertools
@@ -23,7 +24,7 @@ from ..utils import (
 
 
 class XHamsterIE(InfoExtractor):
-    _DOMAINS = r'(?:xhamster\.(?:com|one|desi)|xhms\.pro|xhamster\d+\.com)'
+    _DOMAINS = r'(?:xhamster\.(?:com|one|desi)|xhms\.pro|xhamster\d+\.com|xhday\.com)'
     _VALID_URL = r'''(?x)
                     https?://
                         (?:.+?\.)?%s/
@@ -34,7 +35,7 @@ class XHamsterIE(InfoExtractor):
                     ''' % _DOMAINS
     _TESTS = [{
         'url': 'https://xhamster.com/videos/femaleagent-shy-beauty-takes-the-bait-1509445',
-        'md5': '98b4687efb1ffd331c4197854dc09e8f',
+        'md5': '34e1ab926db5dc2750fed9e1f34304bb',
         'info_dict': {
             'id': '1509445',
             'display_id': 'femaleagent-shy-beauty-takes-the-bait',
@@ -43,6 +44,7 @@ class XHamsterIE(InfoExtractor):
             'timestamp': 1350194821,
             'upload_date': '20121014',
             'uploader': 'Ruseful2011',
+            'uploader_id': 'ruseful2011',
             'duration': 893,
             'age_limit': 18,
         },
@@ -72,6 +74,7 @@ class XHamsterIE(InfoExtractor):
             'timestamp': 1454948101,
             'upload_date': '20160208',
             'uploader': 'parejafree',
+            'uploader_id': 'parejafree',
             'duration': 72,
             'age_limit': 18,
         },
@@ -117,6 +120,9 @@ class XHamsterIE(InfoExtractor):
     }, {
         'url': 'http://de.xhamster.com/videos/skinny-girl-fucks-herself-hard-in-the-forest-xhnBJZx',
         'only_matching': True,
+    }, {
+        'url': 'https://xhday.com/videos/strapless-threesome-xhh7yVf',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -245,6 +251,7 @@ class XHamsterIE(InfoExtractor):
             else:
                 categories = None
 
+            uploader_url = url_or_none(try_get(video, lambda x: x['author']['pageURL']))
             return {
                 'id': video_id,
                 'display_id': display_id,
@@ -253,6 +260,8 @@ class XHamsterIE(InfoExtractor):
                 'timestamp': int_or_none(video.get('created')),
                 'uploader': try_get(
                     video, lambda x: x['author']['name'], compat_str),
+                'uploader_url': uploader_url,
+                'uploader_id': uploader_url.split('/')[-1] if uploader_url else None,
                 'thumbnail': video.get('thumbURL'),
                 'duration': int_or_none(video.get('duration')),
                 'view_count': int_or_none(video.get('views')),
@@ -261,7 +270,7 @@ class XHamsterIE(InfoExtractor):
                 'dislike_count': int_or_none(try_get(
                     video, lambda x: x['rating']['dislikes'], int)),
                 'comment_count': int_or_none(video.get('views')),
-                'age_limit': age_limit,
+                'age_limit': age_limit if age_limit is not None else 18,
                 'categories': categories,
                 'formats': formats,
             }
@@ -352,6 +361,7 @@ class XHamsterIE(InfoExtractor):
             'description': description,
             'upload_date': upload_date,
             'uploader': uploader,
+            'uploader_id': uploader.lower() if uploader else None,
             'thumbnail': thumbnail,
             'duration': duration,
             'view_count': view_count,
@@ -420,6 +430,9 @@ class XHamsterUserIE(InfoExtractor):
             'id': 'firatkaan',
         },
         'playlist_mincount': 1,
+    }, {
+        'url': 'https://xhday.com/users/mobhunter',
+        'only_matching': True,
     }]
 
     def _entries(self, user_id):

From 11665dd2367a2eefd1ad090828f987fef11226e4 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 15 Jun 2022 18:26:54 +0100
Subject: [PATCH 513/860] [test] Fix linter for
 3aa94d7945dfaa0e04acf2700ffe0e43b00db498

---
 test/test_download.py  | 1 +
 test/test_subtitles.py | 2 +-
 2 files changed, 2 insertions(+), 1 deletion(-)

diff --git a/test/test_download.py b/test/test_download.py
index 0951a171a..6a6673bc2 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -88,6 +88,7 @@ class TestDownload(unittest.TestCase):
 
 # Dynamically generate tests
 
+
 def generator(test_case, tname):
 
     def test_template(self):
diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 4cbc69ccd..1197721ff 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -117,7 +117,7 @@ class TestYoutubeSubtitles(BaseTestSubtitles):
         # el, eo, en, eu, et, es, ru, rw, ro, bn, be, bg, uk, jv,
         # bs, ja, or, xh, co, ca, cy, cs, ps, pt, pa, vi, pl, hy,
         # hr, ht, hu, hmn, hi, ha, mg, uz, ml, mn, mi, mk, ur,
-        # mt, ms, mr, ug, ta, my, af, sw, is, am, 
+        # mt, ms, mr, ug, ta, my, af, sw, is, am,
         #                                         *it*, iw, sv, ar,
         # su, zu, az, id, ig, nl, no, ne, ny, fr, ku, fy, fa, fi,
         # ka, kk, sr, sq, ko, kn, km, st, sk, si, so, sn, sm, sl,

From 8a158a936c8b002ef536e9e2b778ded02c09c0fa Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 14 Jun 2022 19:45:34 +0100
Subject: [PATCH 514/860] [NHK] Use new API URL

---
 youtube_dl/extractor/nhk.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/nhk.py b/youtube_dl/extractor/nhk.py
index 46a800e7e..f43d91cd5 100644
--- a/youtube_dl/extractor/nhk.py
+++ b/youtube_dl/extractor/nhk.py
@@ -8,7 +8,7 @@ from ..utils import urljoin
 
 
 class NhkBaseIE(InfoExtractor):
-    _API_URL_TEMPLATE = 'https://api.nhk.or.jp/nhkworld/%sod%slist/v7a/%s/%s/%s/all%s.json'
+    _API_URL_TEMPLATE = 'https://nwapi.nhk.jp/nhkworld/%sod%slist/v7b/%s/%s/%s/all%s.json'
     _BASE_URL_REGEX = r'https?://www3\.nhk\.or\.jp/nhkworld/(?P<lang>[a-z]{2})/ondemand'
     _TYPE_REGEX = r'/(?P<type>video|audio)/'
 

From a03b9775d544b06a5b4f2aa630214c7c22fc2229 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 26 Jun 2022 14:18:33 +0100
Subject: [PATCH 515/860] [Mediaset] Support player version number in URL
 pattern

Ref: https://github.com/yt-dlp/yt-dlp/issues/4141
---
 youtube_dl/extractor/mediaset.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/mediaset.py b/youtube_dl/extractor/mediaset.py
index 2c16fc9e2..20048c6ab 100644
--- a/youtube_dl/extractor/mediaset.py
+++ b/youtube_dl/extractor/mediaset.py
@@ -24,7 +24,7 @@ class MediasetIE(ThePlatformBaseIE):
                             (?:(?:www|static3)\.)?mediasetplay\.mediaset\.it/
                             (?:
                                 (?:video|on-demand|movie)/(?:[^/]+/)+[^/]+_|
-                                player/index\.html\?.*?\bprogramGuid=
+                                player(?:/v\d+)?/index\.html\?.*?\bprogramGuid=
                             )
                     )(?P<id>[0-9A-Z]{16,})
                     '''
@@ -73,6 +73,10 @@ class MediasetIE(ThePlatformBaseIE):
         # iframe twitter (from http://www.wittytv.it/se-prima-mi-fidavo-zero/)
         'url': 'https://static3.mediasetplay.mediaset.it/player/index.html?appKey=5ad3966b1de1c4000d5cec48&programGuid=FAFU000000665104&id=665104',
         'only_matching': True,
+    }, {
+        # embedUrl (from https://www.wittytv.it/amici/est-ce-que-tu-maimes-gabriele-5-dicembre-copia/)
+        'url': 'https://static3.mediasetplay.mediaset.it/player/v2/index.html?partnerId=wittytv&configId=&programGuid=FD00000000153323&autoplay=true&purl=http://www.wittytv.it/amici/est-ce-que-tu-maimes-gabriele-5-dicembre-copia/',
+        'only_matching': True,
     }, {
         'url': 'mediaset:FAFU000000665924',
         'only_matching': True,

From 090acd58c1d810fbef1bac08d70bbfad9c0a7504 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 3 Jul 2022 20:05:21 +0100
Subject: [PATCH 516/860] [options] Improve be35e53 (--match-/reject-title
 parameter value)

Resolves #31064.
---
 youtube_dl/options.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 6521ad881..f6621ef91 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -270,11 +270,11 @@ def parseOpts(overrideArguments=None):
     selection.add_option(
         '--match-title',
         dest='matchtitle', metavar='REGEX',
-        help='Download only matching titles (case-insensitive regex or sub-string)')
+        help='Download only matching titles (case-insensitive regex or alphanumeric sub-string)')
     selection.add_option(
         '--reject-title',
         dest='rejecttitle', metavar='REGEX',
-        help='Skip download for matching titles (case-insensitive regex or sub-string)')
+        help='Skip download for matching titles (case-insensitive regex or alphanumeric sub-string)')
     selection.add_option(
         '--max-downloads',
         dest='max_downloads', metavar='NUMBER', type=int, default=None,

From 5f5c127ece74e52aa5b49b6d2941cc0f848d3c36 Mon Sep 17 00:00:00 2001
From: Kyraminol Endyeran <kyraminari@gmail.com>
Date: Tue, 12 Jul 2022 01:35:40 +0200
Subject: [PATCH 517/860] [VVVVID] Support video/dash types (#31060)

Resolves #31030.
---
 youtube_dl/extractor/vvvvid.py | 15 +++++++++++++++
 1 file changed, 15 insertions(+)

diff --git a/youtube_dl/extractor/vvvvid.py b/youtube_dl/extractor/vvvvid.py
index bc196f8a0..6a0d4e8f0 100644
--- a/youtube_dl/extractor/vvvvid.py
+++ b/youtube_dl/extractor/vvvvid.py
@@ -64,6 +64,18 @@ class VVVVIDIE(InfoExtractor):
         'params': {
             'skip_download': True,
         },
+    }, {
+        # video_type == 'video/dash'
+        'url': 'https://www.vvvvid.it/show/683/made-in-abyss/1542/693786/nanachi',
+        'info_dict': {
+            'id': '693786',
+            'ext': 'mp4',
+            'title': 'Nanachi',
+        },
+        'params': {
+            'skip_download': True,
+            'format': 'mp4',
+        },
     }, {
         'url': 'https://www.vvvvid.it/show/434/perche-dovrei-guardarlo-di-dario-moccia/437/489048',
         'only_matching': True
@@ -205,6 +217,9 @@ class VVVVIDIE(InfoExtractor):
                 })
                 is_youtube = True
                 break
+            elif video_type == 'video/dash':
+                formats.extend(self._extract_m3u8_formats(
+                    embed_code, video_id, 'mp4', m3u8_id='hls', fatal=False))
             else:
                 formats.extend(self._extract_wowza_formats(
                     'http://sb.top-ix.org/videomg/_definst_/mp4:%s/playlist.m3u8' % embed_code, video_id))

From adb5294177265ba35b45746dbb600965076ed150 Mon Sep 17 00:00:00 2001
From: Wes <morganw@gmail.com>
Date: Fri, 29 Jul 2022 20:10:00 -0500
Subject: [PATCH 518/860] [aenetworks] Update _THEPLATFORM_KEY and
 _THEPLATFORM_SECRET (#29749)

Fixes ytdl-org/youtube-dl#29300
---
 youtube_dl/extractor/aenetworks.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index e55c03fd7..2a1f08e39 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -20,8 +20,8 @@ class AENetworksBaseIE(ThePlatformIE):
             (?:history(?:vault)?|aetv|mylifetime|lifetimemovieclub)\.com|
             fyi\.tv
         )/'''
-    _THEPLATFORM_KEY = 'crazyjava'
-    _THEPLATFORM_SECRET = 's3cr3t'
+    _THEPLATFORM_KEY = '43jXaGRQud'
+    _THEPLATFORM_SECRET = 'S10BPXHMlb'
     _DOMAIN_MAP = {
         'history.com': ('HISTORY', 'history'),
         'aetv.com': ('AETV', 'aetv'),

From deee741fb145360576ceae9d69b1b43db082c404 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 9 Aug 2022 21:05:00 +0100
Subject: [PATCH 519/860] [test, etc] Improve download test logs; also clean up
 some new flake8 issues (#31153)

* [test] Identify testcase errors better
* [test] Identify download errors better
* [extractor/minds] Linter
* [extractor/aes] Linter
---
 test/test_download.py         | 7 +++++--
 youtube_dl/aes.py             | 2 +-
 youtube_dl/extractor/minds.py | 2 +-
 3 files changed, 7 insertions(+), 4 deletions(-)

diff --git a/test/test_download.py b/test/test_download.py
index 6a6673bc2..19936969f 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -33,6 +33,7 @@ from youtube_dl.compat import (
 from youtube_dl.utils import (
     DownloadError,
     ExtractorError,
+    error_to_compat_str,
     format_bytes,
     UnavailableVideoError,
 )
@@ -108,7 +109,7 @@ def generator(test_case, tname):
         for tc in test_cases:
             info_dict = tc.get('info_dict', {})
             if not (info_dict.get('id') and info_dict.get('ext')):
-                raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?')
+                raise Exception('Test definition (%s) requires both \'id\' and \'ext\' keys present to define the output file' % (tname, ))
 
         if 'skip' in test_case:
             print_skipping(test_case['skip'])
@@ -161,7 +162,9 @@ def generator(test_case, tname):
                 except (DownloadError, ExtractorError) as err:
                     # Check if the exception is not a network related one
                     if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503):
-                        raise
+                        msg = getattr(err, 'msg', error_to_compat_str(err))
+                        err.msg = '%s (%s)' % (msg, tname, )
+                        raise err
 
                     if try_num == RETRIES:
                         report_warning('%s failed due to network errors, skipping...' % tname)
diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py
index 461bb6d41..d0de2d93f 100644
--- a/youtube_dl/aes.py
+++ b/youtube_dl/aes.py
@@ -303,7 +303,7 @@ def xor(data1, data2):
 
 
 def rijndael_mul(a, b):
-    if(a == 0 or b == 0):
+    if (a == 0 or b == 0):
         return 0
     return RIJNDAEL_EXP_TABLE[(RIJNDAEL_LOG_TABLE[a] + RIJNDAEL_LOG_TABLE[b]) % 0xFF]
 
diff --git a/youtube_dl/extractor/minds.py b/youtube_dl/extractor/minds.py
index 8e9f0f825..e8fd582aa 100644
--- a/youtube_dl/extractor/minds.py
+++ b/youtube_dl/extractor/minds.py
@@ -78,7 +78,7 @@ class MindsIE(MindsBaseIE):
             else:
                 return self.url_result(entity['perma_url'])
         else:
-            assert(entity['subtype'] == 'video')
+            assert (entity['subtype'] == 'video')
             video_id = entity_id
         # 1080p and webm formats available only on the sources array
         video = self._call_api(

From e6a836d54ca1d3cd02f3ee45ef707a46f23e8291 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 10 Aug 2022 15:37:59 +0100
Subject: [PATCH 520/860] [core] Make `--max-downloads ...` stop immediately on
 reaching the limit

Based on and closes #26638.
---
 youtube_dl/YoutubeDL.py | 10 ++++++----
 1 file changed, 6 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 3895b408f..e77b8d50c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1779,10 +1779,9 @@ class YoutubeDL(object):
 
         assert info_dict.get('_type', 'video') == 'video'
 
-        max_downloads = self.params.get('max_downloads')
-        if max_downloads is not None:
-            if self._num_downloads >= int(max_downloads):
-                raise MaxDownloadsReached()
+        max_downloads = int_or_none(self.params.get('max_downloads')) or float('inf')
+        if self._num_downloads >= max_downloads:
+            raise MaxDownloadsReached()
 
         # TODO: backward compatibility, to be removed
         info_dict['fulltitle'] = info_dict['title']
@@ -2062,6 +2061,9 @@ class YoutubeDL(object):
                     self.report_error('postprocessing: %s' % str(err))
                     return
                 self.record_download_archive(info_dict)
+                # avoid possible nugatory search for further items (PR #26638)
+                if self._num_downloads >= max_downloads:
+                    raise MaxDownloadsReached()
 
     def download(self, url_list):
         """Download a given list of URLs."""

From d231b56717c73ee597d2e077d11b69ed48a1b02d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 14 Aug 2022 18:45:45 +0100
Subject: [PATCH 521/860] [jsinterp] Overhaul JSInterp to handle new YT players
 4c3f79c5, 324f67b9 (#31170)

* back-port from yt-dlp 8f53dc44a0cc1c2d98c35740b9293462c080f5d0, thanks pukkandan
* also support void, improve <</>> precedence, improve expressions in comma-list
* add more tests
---
 test/test_jsinterp.py          |  49 ++-
 test/test_utils.py             |   3 +
 test/test_youtube_signature.py |  13 +
 youtube_dl/compat.py           |  54 ++-
 youtube_dl/jsinterp.py         | 581 ++++++++++++++++++++-------------
 youtube_dl/utils.py            |  47 ++-
 6 files changed, 500 insertions(+), 247 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index acdabffb1..c6c931743 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -19,6 +19,9 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('function x3(){return 42;}')
         self.assertEqual(jsi.call_function('x3'), 42)
 
+        jsi = JSInterpreter('function x3(){42}')
+        self.assertEqual(jsi.call_function('x3'), None)
+
         jsi = JSInterpreter('var x5 = function(){return 42;}')
         self.assertEqual(jsi.call_function('x5'), 42)
 
@@ -51,8 +54,11 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('function f(){return 11 >> 2;}')
         self.assertEqual(jsi.call_function('f'), 2)
 
+        jsi = JSInterpreter('function f(){return []? 2+3: 4;}')
+        self.assertEqual(jsi.call_function('f'), 5)
+
     def test_array_access(self):
-        jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2] = 7; return x;}')
+        jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}')
         self.assertEqual(jsi.call_function('f'), [5, 2, 7])
 
     def test_parens(self):
@@ -62,6 +68,10 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('function f(){return (1 + 2) * 3;}')
         self.assertEqual(jsi.call_function('f'), 9)
 
+    def test_quotes(self):
+        jsi = JSInterpreter(r'function f(){return "a\"\\("}')
+        self.assertEqual(jsi.call_function('f'), r'a"\(')
+
     def test_assignments(self):
         jsi = JSInterpreter('function f(){var x = 20; x = 30 + 1; return x;}')
         self.assertEqual(jsi.call_function('f'), 31)
@@ -104,18 +114,29 @@ class TestJSInterpreter(unittest.TestCase):
         }''')
         self.assertEqual(jsi.call_function('x'), [20, 20, 30, 40, 50])
 
+    def test_builtins(self):
+        jsi = JSInterpreter('''
+        function x() { return new Date('Wednesday 31 December 1969 18:01:26 MDT') - 0; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 86000)
+        jsi = JSInterpreter('''
+        function x(dt) { return new Date(dt) - 0; }
+        ''')
+        self.assertEqual(jsi.call_function('x', 'Wednesday 31 December 1969 18:01:26 MDT'), 86000)
+
     def test_call(self):
         jsi = JSInterpreter('''
         function x() { return 2; }
-        function y(a) { return x() + a; }
+        function y(a) { return x() + (a?a:0); }
         function z() { return y(3); }
         ''')
         self.assertEqual(jsi.call_function('z'), 5)
+        self.assertEqual(jsi.call_function('y'), 2)
 
     def test_for_loop(self):
         # function x() { a=0; for (i=0; i-10; i++) {a++} a }
         jsi = JSInterpreter('''
-        function x() { a=0; for (i=0; i-10; i = i + 1) {a++} a }
+        function x() { a=0; for (i=0; i-10; i++) {a++} return a }
         ''')
         self.assertEqual(jsi.call_function('x'), 10)
 
@@ -156,19 +177,19 @@ class TestJSInterpreter(unittest.TestCase):
 
     def test_for_loop_continue(self):
         jsi = JSInterpreter('''
-        function x() { a=0; for (i=0; i-10; i++) { continue; a++ } a }
+        function x() { a=0; for (i=0; i-10; i++) { continue; a++ } return a }
         ''')
         self.assertEqual(jsi.call_function('x'), 0)
 
     def test_for_loop_break(self):
         jsi = JSInterpreter('''
-        function x() { a=0; for (i=0; i-10; i++) { break; a++ } a }
+        function x() { a=0; for (i=0; i-10; i++) { break; a++ } return a }
         ''')
         self.assertEqual(jsi.call_function('x'), 0)
 
     def test_literal_list(self):
         jsi = JSInterpreter('''
-        function x() { [1, 2, "asdf", [5, 6, 7]][3] }
+        function x() { return [1, 2, "asdf", [5, 6, 7]][3] }
         ''')
         self.assertEqual(jsi.call_function('x'), [5, 6, 7])
 
@@ -177,6 +198,22 @@ class TestJSInterpreter(unittest.TestCase):
         function x() { a=5; a -= 1, a+=3; return a }
         ''')
         self.assertEqual(jsi.call_function('x'), 7)
+        jsi = JSInterpreter('''
+        function x() { a=5; return (a -= 1, a+=3, a); }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 7)
+
+    def test_void(self):
+        jsi = JSInterpreter('''
+        function x() { return void 42; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), None)
+
+    def test_return_function(self):
+        jsi = JSInterpreter('''
+        function x() { return [1, function(){return 1}][1] }
+        ''')
+        self.assertEqual(jsi.call_function('x')([]), 1)
 
 
 if __name__ == '__main__':
diff --git a/test/test_utils.py b/test/test_utils.py
index 259c4763e..f1a748dde 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -370,6 +370,9 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(unified_timestamp('Sep 11, 2013 | 5:49 AM'), 1378878540)
         self.assertEqual(unified_timestamp('December 15, 2017 at 7:49 am'), 1513324140)
         self.assertEqual(unified_timestamp('2018-03-14T08:32:43.1493874+00:00'), 1521016363)
+        self.assertEqual(unified_timestamp('December 31 1969 20:00:01 EDT'), 1)
+        self.assertEqual(unified_timestamp('Wednesday 31 December 1969 18:01:26 MDT'), 86)
+        self.assertEqual(unified_timestamp('12/31/1969 20:01:18 EDT', False), 78)
 
     def test_determine_ext(self):
         self.assertEqual(determine_ext('http://example.com/foo/bar.mp4/?download'), 'mp4')
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index fc5e9828e..6e955e0f0 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -90,12 +90,25 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/e06dea74/player_ias.vflset/en_US/base.js',
         'AiuodmaDDYw8d3y4bf', 'ankd8eza2T6Qmw',
     ),
+    (
+        'https://www.youtube.com/s/player/5dd88d1d/player-plasma-ias-phone-en_US.vflset/base.js',
+        'kSxKFLeqzv_ZyHSAt', 'n8gS8oRlHOxPFA',
+    ),
+    (
+        'https://www.youtube.com/s/player/324f67b9/player_ias.vflset/en_US/base.js',
+        'xdftNy7dh9QGnhW', '22qLGxrmX8F1rA',
+    ),
+    (
+        'https://www.youtube.com/s/player/4c3f79c5/player_ias.vflset/en_US/base.js',
+        'TDCstCG66tEAO5pR9o', 'dbxNtZ14c-yWyw',
+    ),
 ]
 
 
 class TestPlayerInfo(unittest.TestCase):
     def test_youtube_extract_player_info(self):
         PLAYER_URLS = (
+            ('https://www.youtube.com/s/player/4c3f79c5/player_ias.vflset/en_US/base.js', '4c3f79c5'),
             ('https://www.youtube.com/s/player/64dddad9/player_ias.vflset/en_US/base.js', '64dddad9'),
             ('https://www.youtube.com/s/player/64dddad9/player_ias.vflset/fr_FR/base.js', '64dddad9'),
             ('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-phone-en_US.vflset/base.js', '64dddad9'),
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 9f5f85dae..6d2c31a61 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2985,7 +2985,6 @@ except ImportError:
     except ImportError:
         compat_filter = filter
 
-
 try:
     from future_builtins import zip as compat_zip
 except ImportError:  # not 2.6+ or is 3.x
@@ -2995,6 +2994,57 @@ except ImportError:  # not 2.6+ or is 3.x
         compat_zip = zip
 
 
+# method renamed between Py2/3
+try:
+    from itertools import zip_longest as compat_itertools_zip_longest
+except ImportError:
+    from itertools import izip_longest as compat_itertools_zip_longest
+
+
+# new class in collections
+try:
+    from collections import ChainMap as compat_collections_chain_map
+except ImportError:
+    # Py < 3.3
+    class compat_collections_chain_map(compat_collections_abc.MutableMapping):
+
+        maps = [{}]
+
+        def __init__(self, *maps):
+            self.maps = list(maps) or [{}]
+
+        def __getitem__(self, k):
+            for m in self.maps:
+                if k in m:
+                    return m[k]
+            raise KeyError(k)
+
+        def __setitem__(self, k, v):
+            self.maps[0].__setitem__(k, v)
+            return
+
+        def __delitem__(self, k):
+            if k in self.maps[0]:
+                del self.maps[0][k]
+                return
+            raise KeyError(k)
+
+        def __iter__(self):
+            return itertools.chain(*reversed(self.maps))
+
+        def __len__(self):
+            return len(iter(self))
+
+        def new_child(self, m=None, **kwargs):
+            m = m or {}
+            m.update(kwargs)
+            return compat_collections_chain_map(m, *self.maps)
+
+        @property
+        def parents(self):
+            return compat_collections_chain_map(*(self.maps[1:]))
+
+
 if sys.version_info < (3, 3):
     def compat_b64decode(s, *args, **kwargs):
         if isinstance(s, compat_str):
@@ -3031,6 +3081,7 @@ __all__ = [
     'compat_basestring',
     'compat_chr',
     'compat_collections_abc',
+    'compat_collections_chain_map',
     'compat_cookiejar',
     'compat_cookiejar_Cookie',
     'compat_cookies',
@@ -3051,6 +3102,7 @@ __all__ = [
     'compat_input',
     'compat_integer_types',
     'compat_itertools_count',
+    'compat_itertools_zip_longest',
     'compat_kwargs',
     'compat_map',
     'compat_numeric_types',
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 8eaa911cd..c60a9b3c2 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -1,42 +1,87 @@
 from __future__ import unicode_literals
 
+import itertools
 import json
+import math
 import operator
 import re
 
 from .utils import (
+    NO_DEFAULT,
     ExtractorError,
+    js_to_json,
     remove_quotes,
+    unified_timestamp,
 )
 from .compat import (
-    compat_collections_abc,
+    compat_collections_chain_map as ChainMap,
+    compat_itertools_zip_longest as zip_longest,
     compat_str,
 )
-MutableMapping = compat_collections_abc.MutableMapping
 
+_NAME_RE = r'[a-zA-Z_$][\w$]*'
 
-class Nonlocal:
-    pass
+# (op, definition) in order of binding priority, tightest first
+# avoid dict to maintain order
+# definition None => Defined in JSInterpreter._operator
+_DOT_OPERATORS = (
+    ('.', None),
+    # TODO: ('?.', None),
+)
 
-
-_OPERATORS = [
+_OPERATORS = (
     ('|', operator.or_),
     ('^', operator.xor),
     ('&', operator.and_),
     ('>>', operator.rshift),
     ('<<', operator.lshift),
-    ('-', operator.sub),
     ('+', operator.add),
-    ('%', operator.mod),
-    ('/', operator.truediv),
+    ('-', operator.sub),
     ('*', operator.mul),
-]
-_ASSIGN_OPERATORS = [(op + '=', opfunc) for op, opfunc in _OPERATORS]
-_ASSIGN_OPERATORS.append(('=', (lambda cur, right: right)))
+    ('/', operator.truediv),
+    ('%', operator.mod),
+)
 
-_NAME_RE = r'[a-zA-Z_$][a-zA-Z_$0-9]*'
+_COMP_OPERATORS = (
+    ('===', operator.is_),
+    ('==', operator.eq),
+    ('!==', operator.is_not),
+    ('!=', operator.ne),
+    ('<=', operator.le),
+    ('>=', operator.ge),
+    ('<', operator.lt),
+    ('>', operator.gt),
+)
+
+_LOG_OPERATORS = (
+    ('&', operator.and_),
+    ('|', operator.or_),
+    ('^', operator.xor),
+)
+
+_SC_OPERATORS = (
+    ('?', None),
+    ('||', None),
+    ('&&', None),
+    # TODO: ('??', None),
+)
+
+_OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS))
 
 _MATCHING_PARENS = dict(zip(*zip('()', '{}', '[]')))
+_QUOTES = '\'"'
+
+
+def _ternary(cndn, if_true=True, if_false=False):
+    """Simulate JS's ternary operator (cndn?if_true:if_false)"""
+    if cndn in (False, None, 0, ''):
+        return if_false
+    try:
+        if math.isnan(cndn):  # NB: NaN cannot be checked by membership
+            return if_false
+    except TypeError:
+        pass
+    return if_true
 
 
 class JS_Break(ExtractorError):
@@ -49,70 +94,77 @@ class JS_Continue(ExtractorError):
         ExtractorError.__init__(self, 'Invalid continue')
 
 
-class LocalNameSpace(MutableMapping):
-    def __init__(self, *stack):
-        self.stack = tuple(stack)
-
-    def __getitem__(self, key):
-        for scope in self.stack:
-            if key in scope:
-                return scope[key]
-        raise KeyError(key)
-
+class LocalNameSpace(ChainMap):
     def __setitem__(self, key, value):
-        for scope in self.stack:
+        for scope in self.maps:
             if key in scope:
                 scope[key] = value
-                break
-        else:
-            self.stack[0][key] = value
-        return value
+                return
+        self.maps[0][key] = value
 
     def __delitem__(self, key):
         raise NotImplementedError('Deleting is not supported')
 
-    def __iter__(self):
-        for scope in self.stack:
-            for scope_item in iter(scope):
-                yield scope_item
-
-    def __len__(self, key):
-        return len(iter(self))
-
     def __repr__(self):
-        return 'LocalNameSpace%s' % (self.stack, )
+        return 'LocalNameSpace%s' % (self.maps, )
 
 
 class JSInterpreter(object):
+    __named_object_counter = 0
+
     def __init__(self, code, objects=None):
-        if objects is None:
-            objects = {}
-        self.code = code
-        self._functions = {}
-        self._objects = objects
-        self.__named_object_counter = 0
+        self.code, self._functions = code, {}
+        self._objects = {} if objects is None else objects
+
+    class Exception(ExtractorError):
+        def __init__(self, msg, *args, **kwargs):
+            expr = kwargs.pop('expr', None)
+            if expr is not None:
+                msg = '{0} in: {1!r}'.format(msg.rstrip(), expr[:100])
+            super(JSInterpreter.Exception, self).__init__(msg, *args, **kwargs)
 
     def _named_object(self, namespace, obj):
         self.__named_object_counter += 1
-        name = '__youtube_dl_jsinterp_obj%s' % (self.__named_object_counter, )
+        name = '__youtube_dl_jsinterp_obj%d' % (self.__named_object_counter, )
         namespace[name] = obj
         return name
 
     @staticmethod
-    def _separate(expr, delim=',', max_split=None):
+    def _separate(expr, delim=',', max_split=None, skip_delims=None):
         if not expr:
             return
         counters = {k: 0 for k in _MATCHING_PARENS.values()}
-        start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
+        start, splits, pos, skipping, delim_len = 0, 0, 0, 0, len(delim) - 1
+        in_quote, escaping = None, False
         for idx, char in enumerate(expr):
-            if char in _MATCHING_PARENS:
-                counters[_MATCHING_PARENS[char]] += 1
-            elif char in counters:
-                counters[char] -= 1
-            if char != delim[pos] or any(counters.values()):
-                pos = 0
+            if not in_quote:
+                if char in _MATCHING_PARENS:
+                    counters[_MATCHING_PARENS[char]] += 1
+                elif char in counters:
+                    counters[char] -= 1
+            if not escaping:
+                if char in _QUOTES and in_quote in (char, None):
+                    in_quote = None if in_quote else char
+                else:
+                    escaping = in_quote and char == '\\'
+            else:
+                escaping = False
+
+            if char != delim[pos] or any(counters.values()) or in_quote:
+                pos = skipping = 0
                 continue
-            elif pos != delim_len:
+            elif skipping > 0:
+                skipping -= 1
+                continue
+            elif pos == 0 and skip_delims:
+                here = expr[idx:]
+                for s in skip_delims if isinstance(skip_delims, (list, tuple)) else [skip_delims]:
+                    if here.startswith(s) and s:
+                        skipping = len(s) - 1
+                        break
+                if skipping > 0:
+                    continue
+            if pos < delim_len:
                 pos += 1
                 continue
             yield expr[start: idx - delim_len]
@@ -122,61 +174,108 @@ class JSInterpreter(object):
                 break
         yield expr[start:]
 
-    @staticmethod
-    def _separate_at_paren(expr, delim):
-        separated = list(JSInterpreter._separate(expr, delim, 1))
+    @classmethod
+    def _separate_at_paren(cls, expr, delim):
+        separated = list(cls._separate(expr, delim, 1))
+
         if len(separated) < 2:
-            raise ExtractorError('No terminating paren {0} in {1}'.format(delim, expr))
+            raise cls.Exception('No terminating paren {delim} in {expr}'.format(**locals()))
         return separated[0][1:].strip(), separated[1].strip()
 
+    @staticmethod
+    def _all_operators():
+        return itertools.chain(
+            _SC_OPERATORS, _LOG_OPERATORS, _COMP_OPERATORS, _OPERATORS)
+
+    def _operator(self, op, left_val, right_expr, expr, local_vars, allow_recursion):
+        if op in ('||', '&&'):
+            if (op == '&&') ^ _ternary(left_val):
+                return left_val  # short circuiting
+        elif op == '?':
+            right_expr = _ternary(left_val, *self._separate(right_expr, ':', 1))
+
+        right_val = self.interpret_expression(right_expr, local_vars, allow_recursion)
+        opfunc = op and next((v for k, v in self._all_operators() if k == op), None)
+        if not opfunc:
+            return right_val
+
+        try:
+            return opfunc(left_val, right_val)
+        except Exception as e:
+            raise self.Exception('Failed to evaluate {left_val!r} {op} {right_val!r}'.format(**locals()), expr, cause=e)
+
+    def _index(self, obj, idx):
+        if idx == 'length':
+            return len(obj)
+        try:
+            return obj[int(idx)] if isinstance(obj, list) else obj[idx]
+        except Exception as e:
+            raise self.Exception('Cannot get index {idx}'.format(**locals()), expr=repr(obj), cause=e)
+
+    def _dump(self, obj, namespace):
+        try:
+            return json.dumps(obj)
+        except TypeError:
+            return self._named_object(namespace, obj)
+
     def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if allow_recursion < 0:
-            raise ExtractorError('Recursion limit reached')
+            raise self.Exception('Recursion limit reached')
+        allow_recursion -= 1
 
-        sub_statements = list(self._separate(stmt, ';'))
-        stmt = (sub_statements or ['']).pop()
+        should_return = False
+        sub_statements = list(self._separate(stmt, ';')) or ['']
+        expr = stmt = sub_statements.pop().strip()
         for sub_stmt in sub_statements:
-            ret, should_abort = self.interpret_statement(sub_stmt, local_vars, allow_recursion - 1)
-            if should_abort:
-                return ret
+            ret, should_return = self.interpret_statement(sub_stmt, local_vars, allow_recursion)
+            if should_return:
+                return ret, should_return
 
-        should_abort = False
-        stmt = stmt.lstrip()
-        stmt_m = re.match(r'var\s', stmt)
-        if stmt_m:
-            expr = stmt[len(stmt_m.group(0)):]
-        else:
-            return_m = re.match(r'return(?:\s+|$)', stmt)
-            if return_m:
-                expr = stmt[len(return_m.group(0)):]
-                should_abort = True
+        m = re.match(r'(?P<var>(?:var|const|let)\s)|return(?:\s+|$)', stmt)
+        if m:
+            expr = stmt[len(m.group(0)):].strip()
+            should_return = not m.group('var')
+        if not expr:
+            return None, should_return
+
+        if expr[0] in _QUOTES:
+            inner, outer = self._separate(expr, expr[0], 1)
+            inner = json.loads(js_to_json(inner + expr[0]))  # , strict=True))
+            if not outer:
+                return inner, should_return
+            expr = self._named_object(local_vars, inner) + outer
+
+        if expr.startswith('new '):
+            obj = expr[4:]
+            if obj.startswith('Date('):
+                left, right = self._separate_at_paren(obj[4:], ')')
+                left = self.interpret_expression(left, local_vars, allow_recursion)
+                expr = unified_timestamp(left, False)
+                if not expr:
+                    raise self.Exception('Failed to parse date {left!r}'.format(**locals()), expr=expr)
+                expr = self._dump(int(expr * 1000), local_vars) + right
             else:
-                # Try interpreting it as an expression
-                expr = stmt
+                raise self.Exception('Unsupported object {obj}'.format(**locals()), expr=expr)
 
-        v = self.interpret_expression(expr, local_vars, allow_recursion)
-        return v, should_abort
-
-    def interpret_expression(self, expr, local_vars, allow_recursion):
-        expr = expr.strip()
-        if expr == '':  # Empty expression
-            return None
+        if expr.startswith('void '):
+            left = self.interpret_expression(expr[5:], local_vars, allow_recursion)
+            return None, should_return
 
         if expr.startswith('{'):
             inner, outer = self._separate_at_paren(expr, '}')
-            inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion - 1)
+            inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion)
             if not outer or should_abort:
-                return inner
+                return inner, should_abort or should_return
             else:
-                expr = json.dumps(inner) + outer
+                expr = self._dump(inner, local_vars) + outer
 
         if expr.startswith('('):
             inner, outer = self._separate_at_paren(expr, ')')
-            inner = self.interpret_expression(inner, local_vars, allow_recursion)
-            if not outer:
-                return inner
+            inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion)
+            if not outer or should_abort:
+                return inner, should_abort or should_return
             else:
-                expr = json.dumps(inner) + outer
+                expr = self._dump(inner, local_vars) + outer
 
         if expr.startswith('['):
             inner, outer = self._separate_at_paren(expr, ']')
@@ -185,57 +284,53 @@ class JSInterpreter(object):
                 for item in self._separate(inner)])
             expr = name + outer
 
-        m = re.match(r'try\s*', expr)
-        if m:
+        m = re.match(r'(?P<try>try|finally)\s*|(?:(?P<catch>catch)|(?P<for>for)|(?P<switch>switch))\s*\(', expr)
+        md = m.groupdict() if m else {}
+        if md.get('try'):
             if expr[m.end()] == '{':
                 try_expr, expr = self._separate_at_paren(expr[m.end():], '}')
             else:
                 try_expr, expr = expr[m.end() - 1:], ''
-            ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion - 1)
+            ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion)
             if should_abort:
-                return ret
-            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+                return ret, True
+            ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
+            return ret, should_abort or should_return
 
-        m = re.match(r'(?:(?P<catch>catch)|(?P<for>for)|(?P<switch>switch))\s*\(', expr)
-        md = m.groupdict() if m else {}
-        if md.get('catch'):
+        elif md.get('catch'):
             # We ignore the catch block
             _, expr = self._separate_at_paren(expr, '}')
-            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+            ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
+            return ret, should_abort or should_return
 
         elif md.get('for'):
-            def raise_constructor_error(c):
-                raise ExtractorError(
-                    'Premature return in the initialization of a for loop in {0!r}'.format(c))
-
             constructor, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
             if remaining.startswith('{'):
                 body, expr = self._separate_at_paren(remaining, '}')
             else:
-                m = re.match(r'switch\s*\(', remaining)  # FIXME
-                if m:
-                    switch_val, remaining = self._separate_at_paren(remaining[m.end() - 1:], ')')
+                switch_m = re.match(r'switch\s*\(', remaining)  # FIXME
+                if switch_m:
+                    switch_val, remaining = self._separate_at_paren(remaining[switch_m.end() - 1:], ')')
                     body, expr = self._separate_at_paren(remaining, '}')
                     body = 'switch(%s){%s}' % (switch_val, body)
                 else:
                     body, expr = remaining, ''
             start, cndn, increment = self._separate(constructor, ';')
-            if self.interpret_statement(start, local_vars, allow_recursion - 1)[1]:
-                raise_constructor_error(constructor)
+            self.interpret_expression(start, local_vars, allow_recursion)
             while True:
-                if not self.interpret_expression(cndn, local_vars, allow_recursion):
+                if not _ternary(self.interpret_expression(cndn, local_vars, allow_recursion)):
                     break
                 try:
-                    ret, should_abort = self.interpret_statement(body, local_vars, allow_recursion - 1)
+                    ret, should_abort = self.interpret_statement(body, local_vars, allow_recursion)
                     if should_abort:
-                        return ret
+                        return ret, True
                 except JS_Break:
                     break
                 except JS_Continue:
                     pass
-                if self.interpret_statement(increment, local_vars, allow_recursion - 1)[1]:
-                    raise_constructor_error(constructor)
-            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+                self.interpret_expression(increment, local_vars, allow_recursion)
+            ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
+            return ret, should_abort or should_return
 
         elif md.get('switch'):
             switch_val, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
@@ -245,7 +340,7 @@ class JSInterpreter(object):
             for default in (False, True):
                 matched = False
                 for item in items:
-                    case, stmt = [i.strip() for i in self._separate(item, ':', 1)]
+                    case, stmt = (i.strip() for i in self._separate(item, ':', 1))
                     if default:
                         matched = matched or case == 'default'
                     elif not matched:
@@ -254,24 +349,28 @@ class JSInterpreter(object):
                     if not matched:
                         continue
                     try:
-                        ret, should_abort = self.interpret_statement(stmt, local_vars, allow_recursion - 1)
+                        ret, should_abort = self.interpret_statement(stmt, local_vars, allow_recursion)
                         if should_abort:
                             return ret
                     except JS_Break:
                         break
                 if matched:
                     break
-            return self.interpret_statement(expr, local_vars, allow_recursion - 1)[0]
+            ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
+            return ret, should_abort or should_return
 
         # Comma separated statements
         sub_expressions = list(self._separate(expr))
-        expr = sub_expressions.pop().strip() if sub_expressions else ''
-        for sub_expr in sub_expressions:
-            self.interpret_expression(sub_expr, local_vars, allow_recursion)
+        if len(sub_expressions) > 1:
+            for sub_expr in sub_expressions:
+                ret, should_abort = self.interpret_statement(sub_expr, local_vars, allow_recursion)
+                if should_abort:
+                    return ret, True
+            return ret, False
 
         for m in re.finditer(r'''(?x)
-                (?P<pre_sign>\+\+|--)(?P<var1>%(_NAME_RE)s)|
-                (?P<var2>%(_NAME_RE)s)(?P<post_sign>\+\+|--)''' % globals(), expr):
+                (?P<pre_sign>\+\+|--)(?P<var1>{_NAME_RE})|
+                (?P<var2>{_NAME_RE})(?P<post_sign>\+\+|--)'''.format(**globals()), expr):
             var = m.group('var1') or m.group('var2')
             start, end = m.span()
             sign = m.group('pre_sign') or m.group('post_sign')
@@ -279,85 +378,87 @@ class JSInterpreter(object):
             local_vars[var] += 1 if sign[0] == '+' else -1
             if m.group('pre_sign'):
                 ret = local_vars[var]
-            expr = expr[:start] + json.dumps(ret) + expr[end:]
+            expr = expr[:start] + self._dump(ret, local_vars) + expr[end:]
 
-        for op, opfunc in _ASSIGN_OPERATORS:
-            m = re.match(r'''(?x)
-                (?P<out>%s)(?:\[(?P<index>[^\]]+?)\])?
-                \s*%s
-                (?P<expr>.*)$''' % (_NAME_RE, re.escape(op)), expr)
-            if not m:
-                continue
-            right_val = self.interpret_expression(m.group('expr'), local_vars, allow_recursion)
+        if not expr:
+            return None, should_return
 
-            if m.groupdict().get('index'):
-                lvar = local_vars[m.group('out')]
-                idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
-                if not isinstance(idx, int):
-                    raise ExtractorError('List indices must be integers: %s' % (idx, ))
-                cur = lvar[idx]
-                val = opfunc(cur, right_val)
-                lvar[idx] = val
-                return val
-            else:
-                cur = local_vars.get(m.group('out'))
-                val = opfunc(cur, right_val)
-                local_vars[m.group('out')] = val
-                return val
+        m = re.match(r'''(?x)
+            (?P<assign>
+                (?P<out>{_NAME_RE})(?:\[(?P<index>[^\]]+?)\])?\s*
+                (?P<op>{_OPERATOR_RE})?
+                =(?P<expr>.*)$
+            )|(?P<return>
+                (?!if|return|true|false|null|undefined)(?P<name>{_NAME_RE})$
+            )|(?P<indexing>
+                (?P<in>{_NAME_RE})\[(?P<idx>.+)\]$
+            )|(?P<attribute>
+                (?P<var>{_NAME_RE})(?:\.(?P<member>[^(]+)|\[(?P<member2>[^\]]+)\])\s*
+            )|(?P<function>
+                (?P<fname>{_NAME_RE})\((?P<args>.*)\)$
+            )'''.format(**globals()), expr)
+        md = m.groupdict() if m else {}
+        if md.get('assign'):
+            left_val = local_vars.get(m.group('out'))
 
-        if expr.isdigit():
-            return int(expr)
+            if not m.group('index'):
+                local_vars[m.group('out')] = self._operator(
+                    m.group('op'), left_val, m.group('expr'), expr, local_vars, allow_recursion)
+                return local_vars[m.group('out')], should_return
+            elif left_val is None:
+                raise self.Exception('Cannot index undefined variable ' + m.group('out'), expr=expr)
 
-        if expr == 'break':
+            idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
+            if not isinstance(idx, (int, float)):
+                raise self.Exception('List index %s must be integer' % (idx, ), expr=expr)
+            idx = int(idx)
+            left_val[idx] = self._operator(
+                m.group('op'), left_val[idx], m.group('expr'), expr, local_vars, allow_recursion)
+            return left_val[idx], should_return
+
+        elif expr.isdigit():
+            return int(expr), should_return
+
+        elif expr == 'break':
             raise JS_Break()
         elif expr == 'continue':
             raise JS_Continue()
 
-        var_m = re.match(
-            r'(?!if|return|true|false|null)(?P<name>%s)$' % _NAME_RE,
-            expr)
-        if var_m:
-            return local_vars[var_m.group('name')]
+        elif md.get('return'):
+            return local_vars[m.group('name')], should_return
 
         try:
-            return json.loads(expr)
+            ret = json.loads(js_to_json(expr))  # strict=True)
+            if not md.get('attribute'):
+                return ret, should_return
         except ValueError:
             pass
 
-        m = re.match(
-            r'(?P<in>%s)\[(?P<idx>.+)\]$' % _NAME_RE, expr)
-        if m:
+        if md.get('indexing'):
             val = local_vars[m.group('in')]
             idx = self.interpret_expression(m.group('idx'), local_vars, allow_recursion)
-            return val[idx]
+            return self._index(val, idx), should_return
 
-        def raise_expr_error(where, op, exp):
-            raise ExtractorError('Premature {0} return of {1} in {2!r}'.format(where, op, exp))
-
-        for op, opfunc in _OPERATORS:
-            separated = list(self._separate(expr, op))
+        for op, _ in self._all_operators():
+            # hackety: </> have higher priority than <</>>, but don't confuse them
+            skip_delim = (op + op) if op in ('<', '>') else None
+            separated = list(self._separate(expr, op, skip_delims=skip_delim))
             if len(separated) < 2:
                 continue
-            right_val = separated.pop()
-            left_val = op.join(separated)
-            left_val, should_abort = self.interpret_statement(
-                left_val, local_vars, allow_recursion - 1)
-            if should_abort:
-                raise_expr_error('left-side', op, expr)
-            right_val, should_abort = self.interpret_statement(
-                right_val, local_vars, allow_recursion - 1)
-            if should_abort:
-                raise_expr_error('right-side', op, expr)
-            return opfunc(left_val or 0, right_val)
 
-        m = re.match(
-            r'(?P<var>%s)(?:\.(?P<member>[^(]+)|\[(?P<member2>[^]]+)\])\s*' % _NAME_RE,
-            expr)
-        if m:
+            right_expr = separated.pop()
+            while op == '-' and len(separated) > 1 and not separated[-1].strip():
+                right_expr = '-' + right_expr
+                separated.pop()
+            left_val = self.interpret_expression(op.join(separated), local_vars, allow_recursion)
+            return self._operator(op, 0 if left_val is None else left_val,
+                                  right_expr, expr, local_vars, allow_recursion), should_return
+
+        if md.get('attribute'):
             variable = m.group('var')
-            nl = Nonlocal()
-
-            nl.member = remove_quotes(m.group('member') or m.group('member2'))
+            member = m.group('member')
+            if not member:
+                member = self.interpret_expression(m.group('member2'), local_vars, allow_recursion)
             arg_str = expr[m.end():]
             if arg_str.startswith('('):
                 arg_str, remaining = self._separate_at_paren(arg_str, ')')
@@ -367,25 +468,24 @@ class JSInterpreter(object):
             def assertion(cndn, msg):
                 """ assert, but without risk of getting optimized out """
                 if not cndn:
-                    raise ExtractorError('{0} {1}: {2}'.format(nl.member, msg, expr))
+                    raise ExtractorError('{member} {msg}'.format(**locals()), expr=expr)
 
             def eval_method():
-                # nonlocal member
-                member = nl.member
-                if variable == 'String':
-                    obj = compat_str
-                elif variable in local_vars:
-                    obj = local_vars[variable]
-                else:
+                if (variable, member) == ('console', 'debug'):
+                    return
+                types = {
+                    'String': compat_str,
+                    'Math': float,
+                }
+                obj = local_vars.get(variable, types.get(variable, NO_DEFAULT))
+                if obj is NO_DEFAULT:
                     if variable not in self._objects:
                         self._objects[variable] = self.extract_object(variable)
                     obj = self._objects[variable]
 
+                # Member access
                 if arg_str is None:
-                    # Member access
-                    if member == 'length':
-                        return len(obj)
-                    return obj[member]
+                    return self._index(obj, member)
 
                 # Function call
                 argvals = [
@@ -396,12 +496,17 @@ class JSInterpreter(object):
                     if member == 'fromCharCode':
                         assertion(argvals, 'takes one or more arguments')
                         return ''.join(map(chr, argvals))
-                    raise ExtractorError('Unsupported string method %s' % (member, ))
+                    raise self.Exception('Unsupported string method ' + member, expr=expr)
+                elif obj == float:
+                    if member == 'pow':
+                        assertion(len(argvals) == 2, 'takes two arguments')
+                        return argvals[0] ** argvals[1]
+                    raise self.Exception('Unsupported Math method ' + member, expr=expr)
 
                 if member == 'split':
                     assertion(argvals, 'takes one or more arguments')
-                    assertion(argvals == [''], 'with arguments is not implemented')
-                    return list(obj)
+                    assertion(len(argvals) == 1, 'with limit argument is not implemented')
+                    return obj.split(argvals[0]) if argvals[0] else list(obj)
                 elif member == 'join':
                     assertion(isinstance(obj, list), 'must be applied on a list')
                     assertion(len(argvals) == 1, 'takes exactly one argument')
@@ -447,7 +552,7 @@ class JSInterpreter(object):
                     assertion(argvals, 'takes one or more arguments')
                     assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
                     f, this = (argvals + [''])[:2]
-                    return [f((item, idx, obj), this=this) for idx, item in enumerate(obj)]
+                    return [f((item, idx, obj), {'this': this}, allow_recursion) for idx, item in enumerate(obj)]
                 elif member == 'indexOf':
                     assertion(argvals, 'takes one or more arguments')
                     assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
@@ -457,32 +562,35 @@ class JSInterpreter(object):
                     except ValueError:
                         return -1
 
-                if isinstance(obj, list):
-                    member = int(member)
-                    nl.member = member
-                return obj[member](argvals)
+                idx = int(member) if isinstance(obj, list) else member
+                return obj[idx](argvals, allow_recursion=allow_recursion)
 
             if remaining:
-                return self.interpret_expression(
+                ret, should_abort = self.interpret_statement(
                     self._named_object(local_vars, eval_method()) + remaining,
                     local_vars, allow_recursion)
+                return ret, should_return or should_abort
             else:
-                return eval_method()
+                return eval_method(), should_return
 
-        m = re.match(r'^(?P<func>%s)\((?P<args>[a-zA-Z0-9_$,]*)\)$' % _NAME_RE, expr)
-        if m:
-            fname = m.group('func')
-            argvals = tuple([
-                int(v) if v.isdigit() else local_vars[v]
-                for v in self._separate(m.group('args'))])
+        elif md.get('function'):
+            fname = m.group('fname')
+            argvals = [self.interpret_expression(v, local_vars, allow_recursion)
+                       for v in self._separate(m.group('args'))]
             if fname in local_vars:
-                return local_vars[fname](argvals)
+                return local_vars[fname](argvals, allow_recursion=allow_recursion), should_return
             elif fname not in self._functions:
                 self._functions[fname] = self.extract_function(fname)
-            return self._functions[fname](argvals)
+            return self._functions[fname](argvals, allow_recursion=allow_recursion), should_return
 
-        if expr:
-            raise ExtractorError('Unsupported JS expression %r' % expr)
+        raise self.Exception(
+            'Unsupported JS expression ' + (expr[:40] if expr != stmt else ''), expr=stmt)
+
+    def interpret_expression(self, expr, local_vars, allow_recursion):
+        ret, should_return = self.interpret_statement(expr, local_vars, allow_recursion)
+        if should_return:
+            raise self.Exception('Cannot return from an expression', expr)
+        return ret
 
     def extract_object(self, objname):
         _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
@@ -494,15 +602,17 @@ class JSInterpreter(object):
                 }\s*;
             ''' % (re.escape(objname), _FUNC_NAME_RE),
             self.code)
+        if not obj_m:
+            raise self.Exception('Could not find object ' + objname)
         fields = obj_m.group('fields')
         # Currently, it only supports function definitions
         fields_m = re.finditer(
             r'''(?x)
-                (?P<key>%s)\s*:\s*function\s*\((?P<args>[a-z,]+)\){(?P<code>[^}]+)}
-            ''' % _FUNC_NAME_RE,
+                (?P<key>%s)\s*:\s*function\s*\((?P<args>(?:%s|,)*)\){(?P<code>[^}]+)}
+            ''' % (_FUNC_NAME_RE, _NAME_RE),
             fields)
         for f in fields_m:
-            argnames = f.group('args').split(',')
+            argnames = self.build_arglist(f.group('args'))
             obj[remove_quotes(f.group('key'))] = self.build_function(argnames, f.group('code'))
 
         return obj
@@ -510,15 +620,19 @@ class JSInterpreter(object):
     def extract_function_code(self, funcname):
         """ @returns argnames, code """
         func_m = re.search(
-            r'''(?x)
-                (?:function\s+%(f_n)s|[{;,]\s*%(f_n)s\s*=\s*function|var\s+%(f_n)s\s*=\s*function)\s*
+            r'''(?xs)
+                (?:
+                    function\s+%(name)s|
+                    [{;,]\s*%(name)s\s*=\s*function|
+                    (?:var|const|let)\s+%(name)s\s*=\s*function
+                )\s*
                 \((?P<args>[^)]*)\)\s*
-                (?P<code>\{(?:(?!};)[^"]|"([^"]|\\")*")+\})''' % {'f_n': re.escape(funcname), },
+                (?P<code>{.+})''' % {'name': re.escape(funcname)},
             self.code)
         code, _ = self._separate_at_paren(func_m.group('code'), '}')  # refine the match
         if func_m is None:
-            raise ExtractorError('Could not find JS function %r' % funcname)
-        return func_m.group('args').split(','), code
+            raise self.Exception('Could not find JS function "{funcname}"'.format(**locals()))
+        return self.build_arglist(func_m.group('args')), code
 
     def extract_function(self, funcname):
         return self.extract_function_from_code(*self.extract_function_code(funcname))
@@ -534,7 +648,7 @@ class JSInterpreter(object):
             name = self._named_object(
                 local_vars,
                 self.extract_function_from_code(
-                    [x.strip() for x in mobj.group('args').split(',')],
+                    self.build_arglist(mobj.group('args')),
                     body, local_vars, *global_stack))
             code = code[:start] + name + remaining
         return self.build_function(argnames, code, local_vars, *global_stack)
@@ -542,17 +656,22 @@ class JSInterpreter(object):
     def call_function(self, funcname, *args):
         return self.extract_function(funcname)(args)
 
+    @classmethod
+    def build_arglist(cls, arg_text):
+        if not arg_text:
+            return []
+        return list(filter(None, (x.strip() or None for x in cls._separate(arg_text))))
+
     def build_function(self, argnames, code, *global_stack):
         global_stack = list(global_stack) or [{}]
-        local_vars = global_stack.pop(0)
+        argnames = tuple(argnames)
 
-        def resf(args, **kwargs):
-            local_vars.update(dict(zip(argnames, args)))
-            local_vars.update(kwargs)
-            var_stack = LocalNameSpace(local_vars, *global_stack)
-            for stmt in self._separate(code.replace('\n', ''), ';'):
-                ret, should_abort = self.interpret_statement(stmt, var_stack)
-                if should_abort:
-                    break
-            return ret
+        def resf(args, kwargs={}, allow_recursion=100):
+            global_stack[0].update(
+                zip_longest(argnames, args, fillvalue=None))
+            global_stack[0].update(kwargs)
+            var_stack = LocalNameSpace(*global_stack)
+            ret, should_abort = self.interpret_statement(code.replace('\n', ''), var_stack, allow_recursion - 1)
+            if should_abort:
+                return ret
         return resf
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4e00317f1..a5f584ec5 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1696,6 +1696,17 @@ MONTH_NAMES = {
         'juillet', 'août', 'septembre', 'octobre', 'novembre', 'décembre'],
 }
 
+# Timezone names for RFC2822 obs-zone
+# From https://github.com/python/cpython/blob/3.11/Lib/email/_parseaddr.py#L36-L42
+TIMEZONE_NAMES = {
+    'UT': 0, 'UTC': 0, 'GMT': 0, 'Z': 0,
+    'AST': -4, 'ADT': -3,  # Atlantic (used in Canada)
+    'EST': -5, 'EDT': -4,  # Eastern
+    'CST': -6, 'CDT': -5,  # Central
+    'MST': -7, 'MDT': -6,  # Mountain
+    'PST': -8, 'PDT': -7   # Pacific
+}
+
 KNOWN_EXTENSIONS = (
     'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
     'flv', 'f4v', 'f4a', 'f4b',
@@ -1735,12 +1746,17 @@ DATE_FORMATS = (
     '%b %dth %Y %I:%M',
     '%Y %m %d',
     '%Y-%m-%d',
+    '%Y.%m.%d.',
     '%Y/%m/%d',
     '%Y/%m/%d %H:%M',
     '%Y/%m/%d %H:%M:%S',
+    '%Y%m%d%H%M',
+    '%Y%m%d%H%M%S',
+    '%Y%m%d',
     '%Y-%m-%d %H:%M',
     '%Y-%m-%d %H:%M:%S',
     '%Y-%m-%d %H:%M:%S.%f',
+    '%Y-%m-%d %H:%M:%S:%f',
     '%d.%m.%Y %H:%M',
     '%d.%m.%Y %H.%M',
     '%Y-%m-%dT%H:%M:%SZ',
@@ -1753,6 +1769,7 @@ DATE_FORMATS = (
     '%b %d %Y at %H:%M:%S',
     '%B %d %Y at %H:%M',
     '%B %d %Y at %H:%M:%S',
+    '%H:%M %d-%b-%Y',
 )
 
 DATE_FORMATS_DAY_FIRST = list(DATE_FORMATS)
@@ -1763,6 +1780,7 @@ DATE_FORMATS_DAY_FIRST.extend([
     '%d/%m/%Y',
     '%d/%m/%y',
     '%d/%m/%Y %H:%M:%S',
+    '%d-%m-%Y %H:%M',
 ])
 
 DATE_FORMATS_MONTH_FIRST = list(DATE_FORMATS)
@@ -2966,10 +2984,22 @@ class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
 
 def extract_timezone(date_str):
     m = re.search(
-        r'^.{8,}?(?P<tz>Z$| ?(?P<sign>\+|-)(?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})$)',
-        date_str)
+        r'''(?x)
+            ^.{8,}?                                              # >=8 char non-TZ prefix, if present
+            (?P<tz>Z|                                            # just the UTC Z, or
+                (?:(?<=.\b\d{4}|\b\d{2}:\d\d)|                   # preceded by 4 digits or hh:mm or
+                   (?<!.\b[a-zA-Z]{3}|[a-zA-Z]{4}|..\b\d\d))     # not preceded by 3 alpha word or >= 4 alpha or 2 digits
+                   [ ]?                                          # optional space
+                (?P<sign>\+|-)                                   # +/-
+                (?P<hours>[0-9]{2}):?(?P<minutes>[0-9]{2})       # hh[:]mm
+            $)
+        ''', date_str)
     if not m:
-        timezone = datetime.timedelta()
+        m = re.search(r'\d{1,2}:\d{1,2}(?:\.\d+)?(?P<tz>\s*[A-Z]+)$', date_str)
+        timezone = TIMEZONE_NAMES.get(m and m.group('tz').strip())
+        if timezone is not None:
+            date_str = date_str[:-len(m.group('tz'))]
+        timezone = datetime.timedelta(hours=timezone or 0)
     else:
         date_str = date_str[:-len(m.group('tz'))]
         if not m.group('sign'):
@@ -3037,7 +3067,8 @@ def unified_timestamp(date_str, day_first=True):
     if date_str is None:
         return None
 
-    date_str = re.sub(r'[,|]', '', date_str)
+    date_str = re.sub(r'\s+', ' ', re.sub(
+        r'(?i)[,|]|(mon|tues?|wed(nes)?|thu(rs)?|fri|sat(ur)?)(day)?', '', date_str))
 
     pm_delta = 12 if re.search(r'(?i)PM', date_str) else 0
     timezone, date_str = extract_timezone(date_str)
@@ -3063,7 +3094,7 @@ def unified_timestamp(date_str, day_first=True):
             pass
     timetuple = email.utils.parsedate_tz(date_str)
     if timetuple:
-        return calendar.timegm(timetuple) + pm_delta * 3600
+        return calendar.timegm(timetuple) + pm_delta * 3600 - timezone.total_seconds()
 
 
 def determine_ext(url, default_ext='unknown_video'):
@@ -3673,13 +3704,11 @@ def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
     if get_attr:
         if v is not None:
             v = getattr(v, get_attr, None)
-    if v == '':
-        v = None
-    if v is None:
+    if v in (None, ''):
         return default
     try:
         return int(v) * invscale // scale
-    except (ValueError, TypeError):
+    except (ValueError, TypeError, OverflowError):
         return default
 
 

From e52e8b8111cf7ca27daef184bacd926865e951b1 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 15 Aug 2022 16:45:04 +0100
Subject: [PATCH 522/860] [postprocessor] Don't replace existing value with
 null metadata parsed from title

---
 youtube_dl/postprocessor/metadatafromtitle.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/postprocessor/metadatafromtitle.py b/youtube_dl/postprocessor/metadatafromtitle.py
index f5c14d974..6cd5bb70f 100644
--- a/youtube_dl/postprocessor/metadatafromtitle.py
+++ b/youtube_dl/postprocessor/metadatafromtitle.py
@@ -40,6 +40,8 @@ class MetadataFromTitlePP(PostProcessor):
                 % self._titleformat)
             return [], info
         for attribute, value in match.groupdict().items():
+            if value is None:
+                continue
             info[attribute] = value
             self._downloader.to_screen(
                 '[fromtitle] parsed %s: %s'

From b0a60ce2032172aeaaf27fe3866ab72768f10cb2 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 17 Aug 2022 14:22:02 +0100
Subject: [PATCH 523/860] [jsinterp] Improve JS language support (#31175)

* operator ??
* operator ?.
* operator **
* accurate operator functions
* `undefined` handling
* object literals {a: 1, "b": expr}
* more tests for weird JS comparisons: see https://github.com/ytdl-org/youtube-dl/issues/31173#issuecomment-1217854397.
---
 test/test_jsinterp.py          | 114 ++++++++++++++++++++
 test/test_youtube_signature.py |   4 +
 youtube_dl/jsinterp.py         | 189 ++++++++++++++++++++++++++-------
 3 files changed, 267 insertions(+), 40 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index c6c931743..328941e09 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -8,7 +8,10 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+import math
+
 from youtube_dl.jsinterp import JSInterpreter
+undefined = JSInterpreter.undefined
 
 
 class TestJSInterpreter(unittest.TestCase):
@@ -48,6 +51,9 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('function f(){return 1 << 5;}')
         self.assertEqual(jsi.call_function('f'), 32)
 
+        jsi = JSInterpreter('function f(){return 2 ** 5}')
+        self.assertEqual(jsi.call_function('f'), 32)
+
         jsi = JSInterpreter('function f(){return 19 & 21;}')
         self.assertEqual(jsi.call_function('f'), 17)
 
@@ -57,6 +63,15 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('function f(){return []? 2+3: 4;}')
         self.assertEqual(jsi.call_function('f'), 5)
 
+        jsi = JSInterpreter('function f(){return 1 == 2}')
+        self.assertEqual(jsi.call_function('f'), False)
+
+        jsi = JSInterpreter('function f(){return 0 && 1 || 2;}')
+        self.assertEqual(jsi.call_function('f'), 2)
+
+        jsi = JSInterpreter('function f(){return 0 ?? 42;}')
+        self.assertEqual(jsi.call_function('f'), 0)
+
     def test_array_access(self):
         jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}')
         self.assertEqual(jsi.call_function('f'), [5, 2, 7])
@@ -203,6 +218,11 @@ class TestJSInterpreter(unittest.TestCase):
         ''')
         self.assertEqual(jsi.call_function('x'), 7)
 
+        jsi = JSInterpreter('''
+        function x() { return (l=[0,1,2,3], function(a, b){return a+b})((l[1], l[2]), l[3]) }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 5)
+
     def test_void(self):
         jsi = JSInterpreter('''
         function x() { return void 42; }
@@ -215,6 +235,100 @@ class TestJSInterpreter(unittest.TestCase):
         ''')
         self.assertEqual(jsi.call_function('x')([]), 1)
 
+    def test_null(self):
+        jsi = JSInterpreter('''
+        function x() { return null; }
+        ''')
+        self.assertIs(jsi.call_function('x'), None)
+
+        jsi = JSInterpreter('''
+        function x() { return [null > 0, null < 0, null == 0, null === 0]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [False, False, False, False])
+
+        jsi = JSInterpreter('''
+        function x() { return [null >= 0, null <= 0]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [True, True])
+
+    def test_undefined(self):
+        jsi = JSInterpreter('''
+        function x() { return undefined === undefined; }
+        ''')
+        self.assertTrue(jsi.call_function('x'))
+
+        jsi = JSInterpreter('''
+        function x() { return undefined; }
+        ''')
+        self.assertIs(jsi.call_function('x'), undefined)
+
+        jsi = JSInterpreter('''
+        function x() { let v; return v; }
+        ''')
+        self.assertIs(jsi.call_function('x'), undefined)
+
+        jsi = JSInterpreter('''
+        function x() { return [undefined === undefined, undefined == undefined, undefined < undefined, undefined > undefined]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [True, True, False, False])
+
+        jsi = JSInterpreter('''
+        function x() { return [undefined === 0, undefined == 0, undefined < 0, undefined > 0]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [False, False, False, False])
+
+        jsi = JSInterpreter('''
+        function x() { return [undefined >= 0, undefined <= 0]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [False, False])
+
+        jsi = JSInterpreter('''
+        function x() { return [undefined > null, undefined < null, undefined == null, undefined === null]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [False, False, True, False])
+
+        jsi = JSInterpreter('''
+        function x() { return [undefined === null, undefined == null, undefined < null, undefined > null]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [False, True, False, False])
+
+        jsi = JSInterpreter('''
+        function x() { let v; return [42+v, v+42, v**42, 42**v, 0**v]; }
+        ''')
+        for y in jsi.call_function('x'):
+            self.assertTrue(math.isnan(y))
+
+        jsi = JSInterpreter('''
+        function x() { let v; return v**0; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 1)
+
+        jsi = JSInterpreter('''
+        function x() { let v; return [v>42, v<=42, v&&42, 42&&v]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [False, False, undefined, undefined])
+
+        jsi = JSInterpreter('function x(){return undefined ?? 42; }')
+        self.assertEqual(jsi.call_function('x'), 42)
+
+    def test_object(self):
+        jsi = JSInterpreter('''
+        function x() { return {}; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), {})
+        jsi = JSInterpreter('''
+        function x() { let a = {m1: 42, m2: 0 }; return [a["m1"], a.m2]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), [42, 0])
+        jsi = JSInterpreter('''
+        function x() { let a; return a?.qq; }
+        ''')
+        self.assertIs(jsi.call_function('x'), undefined)
+        jsi = JSInterpreter('''
+        function x() { let a = {m1: 42, m2: 0 }; return a?.qq; }
+        ''')
+        self.assertIs(jsi.call_function('x'), undefined)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 6e955e0f0..4d756dad3 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -102,6 +102,10 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/4c3f79c5/player_ias.vflset/en_US/base.js',
         'TDCstCG66tEAO5pR9o', 'dbxNtZ14c-yWyw',
     ),
+    (
+        'https://www.youtube.com/s/player/c81bbb4a/player_ias.vflset/en_US/base.js',
+        'gre3EcLurNY2vqp94', 'Z9DfGxWP115WTg',
+    ),
 ]
 
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index c60a9b3c2..8e119d08a 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -7,7 +7,6 @@ import operator
 import re
 
 from .utils import (
-    NO_DEFAULT,
     ExtractorError,
     js_to_json,
     remove_quotes,
@@ -21,6 +20,70 @@ from .compat import (
 
 _NAME_RE = r'[a-zA-Z_$][\w$]*'
 
+_UNDEFINED = object()
+
+
+def _js_bit_op(op):
+
+    def wrapped(a, b):
+        def zeroise(x):
+            return 0 if x in (None, _UNDEFINED) else x
+        return op(zeroise(a), zeroise(b))
+
+    return wrapped
+
+
+def _js_arith_op(op):
+
+    def wrapped(a, b):
+        if _UNDEFINED in (a, b):
+            return float('nan')
+        return op(a or 0, b or 0)
+
+    return wrapped
+
+
+def _js_div(a, b):
+    if _UNDEFINED in (a, b) or not (a and b):
+        return float('nan')
+    return float('inf') if not b else operator.truediv(a or 0, b)
+
+
+def _js_mod(a, b):
+    if _UNDEFINED in (a, b) or not b:
+        return float('nan')
+    return (a or 0) % b
+
+
+def _js_exp(a, b):
+    if not b:
+        # even 0 ** 0 !!
+        return 1
+    if _UNDEFINED in (a, b):
+        return float('nan')
+    return (a or 0) ** b
+
+
+def _js_eq_op(op):
+
+    def wrapped(a, b):
+        if set((a, b)) <= set((None, _UNDEFINED)):
+            return op(a, a)
+        return op(a, b)
+
+    return wrapped
+
+
+def _js_comp_op(op):
+
+    def wrapped(a, b):
+        if _UNDEFINED in (a, b):
+            return False
+        return op(a or 0, b or 0)
+
+    return wrapped
+
+
 # (op, definition) in order of binding priority, tightest first
 # avoid dict to maintain order
 # definition None => Defined in JSInterpreter._operator
@@ -30,40 +93,38 @@ _DOT_OPERATORS = (
 )
 
 _OPERATORS = (
-    ('|', operator.or_),
-    ('^', operator.xor),
-    ('&', operator.and_),
-    ('>>', operator.rshift),
-    ('<<', operator.lshift),
-    ('+', operator.add),
-    ('-', operator.sub),
-    ('*', operator.mul),
-    ('/', operator.truediv),
-    ('%', operator.mod),
+    ('>>', _js_bit_op(operator.rshift)),
+    ('<<', _js_bit_op(operator.lshift)),
+    ('+', _js_arith_op(operator.add)),
+    ('-', _js_arith_op(operator.sub)),
+    ('*', _js_arith_op(operator.mul)),
+    ('/', _js_div),
+    ('%', _js_mod),
+    ('**', _js_exp),
 )
 
 _COMP_OPERATORS = (
     ('===', operator.is_),
-    ('==', operator.eq),
+    ('==', _js_eq_op(operator.eq)),
     ('!==', operator.is_not),
-    ('!=', operator.ne),
-    ('<=', operator.le),
-    ('>=', operator.ge),
-    ('<', operator.lt),
-    ('>', operator.gt),
+    ('!=', _js_eq_op(operator.ne)),
+    ('<=', _js_comp_op(operator.le)),
+    ('>=', _js_comp_op(operator.ge)),
+    ('<', _js_comp_op(operator.lt)),
+    ('>', _js_comp_op(operator.gt)),
 )
 
 _LOG_OPERATORS = (
-    ('&', operator.and_),
-    ('|', operator.or_),
-    ('^', operator.xor),
+    ('|', _js_bit_op(operator.or_)),
+    ('^', _js_bit_op(operator.xor)),
+    ('&', _js_bit_op(operator.and_)),
 )
 
 _SC_OPERATORS = (
     ('?', None),
+    ('??', None),
     ('||', None),
     ('&&', None),
-    # TODO: ('??', None),
 )
 
 _OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS))
@@ -74,7 +135,7 @@ _QUOTES = '\'"'
 
 def _ternary(cndn, if_true=True, if_false=False):
     """Simulate JS's ternary operator (cndn?if_true:if_false)"""
-    if cndn in (False, None, 0, ''):
+    if cndn in (False, None, 0, '', _UNDEFINED):
         return if_false
     try:
         if math.isnan(cndn):  # NB: NaN cannot be checked by membership
@@ -95,6 +156,12 @@ class JS_Continue(ExtractorError):
 
 
 class LocalNameSpace(ChainMap):
+    def __getitem__(self, key):
+        try:
+            return super(LocalNameSpace, self).__getitem__(key)
+        except KeyError:
+            return _UNDEFINED
+
     def __setitem__(self, key, value):
         for scope in self.maps:
             if key in scope:
@@ -105,6 +172,13 @@ class LocalNameSpace(ChainMap):
     def __delitem__(self, key):
         raise NotImplementedError('Deleting is not supported')
 
+    def __contains__(self, key):
+        try:
+            super(LocalNameSpace, self).__getitem__(key)
+            return True
+        except KeyError:
+            return False
+
     def __repr__(self):
         return 'LocalNameSpace%s' % (self.maps, )
 
@@ -112,6 +186,8 @@ class LocalNameSpace(ChainMap):
 class JSInterpreter(object):
     __named_object_counter = 0
 
+    undefined = _UNDEFINED
+
     def __init__(self, code, objects=None):
         self.code, self._functions = code, {}
         self._objects = {} if objects is None else objects
@@ -185,12 +261,16 @@ class JSInterpreter(object):
     @staticmethod
     def _all_operators():
         return itertools.chain(
+            # Ref: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_Precedence
             _SC_OPERATORS, _LOG_OPERATORS, _COMP_OPERATORS, _OPERATORS)
 
     def _operator(self, op, left_val, right_expr, expr, local_vars, allow_recursion):
         if op in ('||', '&&'):
             if (op == '&&') ^ _ternary(left_val):
                 return left_val  # short circuiting
+        elif op == '??':
+            if left_val not in (None, self.undefined):
+                return left_val
         elif op == '?':
             right_expr = _ternary(left_val, *self._separate(right_expr, ':', 1))
 
@@ -204,12 +284,14 @@ class JSInterpreter(object):
         except Exception as e:
             raise self.Exception('Failed to evaluate {left_val!r} {op} {right_val!r}'.format(**locals()), expr, cause=e)
 
-    def _index(self, obj, idx):
+    def _index(self, obj, idx, allow_undefined=False):
         if idx == 'length':
             return len(obj)
         try:
             return obj[int(idx)] if isinstance(obj, list) else obj[idx]
         except Exception as e:
+            if allow_undefined:
+                return self.undefined
             raise self.Exception('Cannot get index {idx}'.format(**locals()), expr=repr(obj), cause=e)
 
     def _dump(self, obj, namespace):
@@ -249,8 +331,8 @@ class JSInterpreter(object):
             obj = expr[4:]
             if obj.startswith('Date('):
                 left, right = self._separate_at_paren(obj[4:], ')')
-                left = self.interpret_expression(left, local_vars, allow_recursion)
-                expr = unified_timestamp(left, False)
+                expr = unified_timestamp(
+                    self.interpret_expression(left, local_vars, allow_recursion), False)
                 if not expr:
                     raise self.Exception('Failed to parse date {left!r}'.format(**locals()), expr=expr)
                 expr = self._dump(int(expr * 1000), local_vars) + right
@@ -263,6 +345,14 @@ class JSInterpreter(object):
 
         if expr.startswith('{'):
             inner, outer = self._separate_at_paren(expr, '}')
+            # try for object expression
+            sub_expressions = [list(self._separate(sub_expr.strip(), ':', 1)) for sub_expr in self._separate(inner)]
+            if all(len(sub_expr) == 2 for sub_expr in sub_expressions):
+                return dict(
+                    (key_expr if re.match(_NAME_RE, key_expr) else key_expr,
+                     self.interpret_expression(val_expr, local_vars, allow_recursion))
+                    for key_expr, val_expr in sub_expressions), should_return
+            # or statement list
             inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion)
             if not outer or should_abort:
                 return inner, should_abort or should_return
@@ -387,13 +477,13 @@ class JSInterpreter(object):
             (?P<assign>
                 (?P<out>{_NAME_RE})(?:\[(?P<index>[^\]]+?)\])?\s*
                 (?P<op>{_OPERATOR_RE})?
-                =(?P<expr>.*)$
+                =(?!=)(?P<expr>.*)$
             )|(?P<return>
                 (?!if|return|true|false|null|undefined)(?P<name>{_NAME_RE})$
             )|(?P<indexing>
                 (?P<in>{_NAME_RE})\[(?P<idx>.+)\]$
             )|(?P<attribute>
-                (?P<var>{_NAME_RE})(?:\.(?P<member>[^(]+)|\[(?P<member2>[^\]]+)\])\s*
+                (?P<var>{_NAME_RE})(?:(?P<nullish>\?)?\.(?P<member>[^(]+)|\[(?P<member2>[^\]]+)\])\s*
             )|(?P<function>
                 (?P<fname>{_NAME_RE})\((?P<args>.*)\)$
             )'''.format(**globals()), expr)
@@ -405,7 +495,7 @@ class JSInterpreter(object):
                 local_vars[m.group('out')] = self._operator(
                     m.group('op'), left_val, m.group('expr'), expr, local_vars, allow_recursion)
                 return local_vars[m.group('out')], should_return
-            elif left_val is None:
+            elif left_val in (None, self.undefined):
                 raise self.Exception('Cannot index undefined variable ' + m.group('out'), expr=expr)
 
             idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
@@ -424,6 +514,9 @@ class JSInterpreter(object):
         elif expr == 'continue':
             raise JS_Continue()
 
+        elif expr == 'undefined':
+            return self.undefined, should_return
+
         elif md.get('return'):
             return local_vars[m.group('name')], should_return
 
@@ -441,7 +534,9 @@ class JSInterpreter(object):
 
         for op, _ in self._all_operators():
             # hackety: </> have higher priority than <</>>, but don't confuse them
-            skip_delim = (op + op) if op in ('<', '>') else None
+            skip_delim = (op + op) if op in '<>*?' else None
+            if op == '?':
+                skip_delim = (skip_delim, '?.')
             separated = list(self._separate(expr, op, skip_delims=skip_delim))
             if len(separated) < 2:
                 continue
@@ -451,12 +546,10 @@ class JSInterpreter(object):
                 right_expr = '-' + right_expr
                 separated.pop()
             left_val = self.interpret_expression(op.join(separated), local_vars, allow_recursion)
-            return self._operator(op, 0 if left_val is None else left_val,
-                                  right_expr, expr, local_vars, allow_recursion), should_return
+            return self._operator(op, left_val, right_expr, expr, local_vars, allow_recursion), should_return
 
         if md.get('attribute'):
-            variable = m.group('var')
-            member = m.group('member')
+            variable, member, nullish = m.group('var', 'member', 'nullish')
             if not member:
                 member = self.interpret_expression(m.group('member2'), local_vars, allow_recursion)
             arg_str = expr[m.end():]
@@ -477,15 +570,24 @@ class JSInterpreter(object):
                     'String': compat_str,
                     'Math': float,
                 }
-                obj = local_vars.get(variable, types.get(variable, NO_DEFAULT))
-                if obj is NO_DEFAULT:
-                    if variable not in self._objects:
-                        self._objects[variable] = self.extract_object(variable)
-                    obj = self._objects[variable]
+                obj = local_vars.get(variable)
+                if obj in (self.undefined, None):
+                    obj = types.get(variable, self.undefined)
+                if obj is self.undefined:
+                    try:
+                        if variable not in self._objects:
+                            self._objects[variable] = self.extract_object(variable)
+                        obj = self._objects[variable]
+                    except self.Exception:
+                        if not nullish:
+                            raise
+
+                if nullish and obj is self.undefined:
+                    return self.undefined
 
                 # Member access
                 if arg_str is None:
-                    return self._index(obj, member)
+                    return self._index(obj, member, nullish)
 
                 # Function call
                 argvals = [
@@ -660,7 +762,14 @@ class JSInterpreter(object):
     def build_arglist(cls, arg_text):
         if not arg_text:
             return []
-        return list(filter(None, (x.strip() or None for x in cls._separate(arg_text))))
+
+        def valid_arg(y):
+            y = y.strip()
+            if not y:
+                raise cls.Exception('Missing arg in "%s"' % (arg_text, ))
+            return y
+
+        return [valid_arg(x) for x in cls._separate(arg_text)]
 
     def build_function(self, argnames, code, *global_stack):
         global_stack = list(global_stack) or [{}]

From 538ec65ba7634bb9ad9f8eb4ce72713c673969dc Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 19 Aug 2022 11:45:04 +0100
Subject: [PATCH 524/860] [jsinterp] Handle regexp literals and throw/catch
 execution (#31182)

* based on https://github.com/yt-dlp/yt-dlp/commit/f6ca640b122239d5ab215f8c2564efb7ac3e8c65, thanks pukkandan
* adds parse support for regexp flags
---
 test/test_jsinterp.py          |  21 +++++
 test/test_youtube_signature.py |   4 +
 youtube_dl/jsinterp.py         | 136 +++++++++++++++++++++++++++------
 3 files changed, 139 insertions(+), 22 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 328941e09..faddf00d5 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -9,6 +9,7 @@ import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import math
+import re
 
 from youtube_dl.jsinterp import JSInterpreter
 undefined = JSInterpreter.undefined
@@ -316,19 +317,39 @@ class TestJSInterpreter(unittest.TestCase):
         function x() { return {}; }
         ''')
         self.assertEqual(jsi.call_function('x'), {})
+
         jsi = JSInterpreter('''
         function x() { let a = {m1: 42, m2: 0 }; return [a["m1"], a.m2]; }
         ''')
         self.assertEqual(jsi.call_function('x'), [42, 0])
+
         jsi = JSInterpreter('''
         function x() { let a; return a?.qq; }
         ''')
         self.assertIs(jsi.call_function('x'), undefined)
+
         jsi = JSInterpreter('''
         function x() { let a = {m1: 42, m2: 0 }; return a?.qq; }
         ''')
         self.assertIs(jsi.call_function('x'), undefined)
 
+    def test_regex(self):
+        jsi = JSInterpreter('''
+        function x() { let a=/,,[/,913,/](,)}/; }
+        ''')
+        self.assertIs(jsi.call_function('x'), None)
+
+        jsi = JSInterpreter('''
+        function x() { let a=/,,[/,913,/](,)}/; return a; }
+        ''')
+        # Pythons disagree on the type of a pattern
+        self.assertTrue(isinstance(jsi.call_function('x'), type(re.compile(''))))
+
+        jsi = JSInterpreter('''
+        function x() { let a=/,,[/,913,/](,)}/i; return a; }
+        ''')
+        self.assertEqual(jsi.call_function('x').flags & re.I, re.I)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 4d756dad3..43e22388d 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -106,6 +106,10 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/c81bbb4a/player_ias.vflset/en_US/base.js',
         'gre3EcLurNY2vqp94', 'Z9DfGxWP115WTg',
     ),
+    (
+        'https://www.youtube.com/s/player/1f7d5369/player_ias.vflset/en_US/base.js',
+        'batNX7sYqIJdkJ', 'IhOkL_zxbkOZBw',
+    ),
 ]
 
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 8e119d08a..48c27a1c0 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -7,6 +7,7 @@ import operator
 import re
 
 from .utils import (
+    error_to_compat_str,
     ExtractorError,
     js_to_json,
     remove_quotes,
@@ -130,7 +131,7 @@ _SC_OPERATORS = (
 _OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS))
 
 _MATCHING_PARENS = dict(zip(*zip('()', '{}', '[]')))
-_QUOTES = '\'"'
+_QUOTES = '\'"/'
 
 
 def _ternary(cndn, if_true=True, if_false=False):
@@ -155,6 +156,12 @@ class JS_Continue(ExtractorError):
         ExtractorError.__init__(self, 'Invalid continue')
 
 
+class JS_Throw(ExtractorError):
+    def __init__(self, e):
+        self.error = e
+        ExtractorError.__init__(self, 'Uncaught exception ' + error_to_compat_str(e))
+
+
 class LocalNameSpace(ChainMap):
     def __getitem__(self, key):
         try:
@@ -172,6 +179,17 @@ class LocalNameSpace(ChainMap):
     def __delitem__(self, key):
         raise NotImplementedError('Deleting is not supported')
 
+    # except
+    def pop(self, key, *args):
+        try:
+            off = self.__getitem__(key)
+            super(LocalNameSpace, self).__delitem__(key)
+            return off
+        except KeyError:
+            if len(args) > 0:
+                return args[0]
+            raise
+
     def __contains__(self, key):
         try:
             super(LocalNameSpace, self).__getitem__(key)
@@ -188,9 +206,29 @@ class JSInterpreter(object):
 
     undefined = _UNDEFINED
 
+    RE_FLAGS = {
+        # special knowledge: Python's re flags are bitmask values, current max 128
+        # invent new bitmask values well above that for literal parsing
+        # TODO: new pattern class to execute matches with these flags
+        'd': 1024,  # Generate indices for substring matches
+        'g': 2048,  # Global search
+        'i': re.I,  # Case-insensitive search
+        'm': re.M,  # Multi-line search
+        's': re.S,  # Allows . to match newline characters
+        'u': re.U,  # Treat a pattern as a sequence of unicode code points
+        'y': 4096,  # Perform a "sticky" search that matches starting at the current position in the target string
+    }
+
+    _EXC_NAME = '__youtube_dl_exception__'
+    _OBJ_NAME = '__youtube_dl_jsinterp_obj'
+
+    OP_CHARS = None
+
     def __init__(self, code, objects=None):
         self.code, self._functions = code, {}
         self._objects = {} if objects is None else objects
+        if type(self).OP_CHARS is None:
+            type(self).OP_CHARS = self.OP_CHARS = self.__op_chars()
 
     class Exception(ExtractorError):
         def __init__(self, msg, *args, **kwargs):
@@ -199,32 +237,64 @@ class JSInterpreter(object):
                 msg = '{0} in: {1!r}'.format(msg.rstrip(), expr[:100])
             super(JSInterpreter.Exception, self).__init__(msg, *args, **kwargs)
 
+    @classmethod
+    def __op_chars(cls):
+        op_chars = set(';,')
+        for op in cls._all_operators():
+            for c in op[0]:
+                op_chars.add(c)
+        return op_chars
+
     def _named_object(self, namespace, obj):
         self.__named_object_counter += 1
-        name = '__youtube_dl_jsinterp_obj%d' % (self.__named_object_counter, )
+        name = '%s%d' % (self._OBJ_NAME, self.__named_object_counter)
         namespace[name] = obj
         return name
 
-    @staticmethod
-    def _separate(expr, delim=',', max_split=None, skip_delims=None):
+    @classmethod
+    def _regex_flags(cls, expr):
+        flags = 0
+        if not expr:
+            return flags, expr
+        for idx, ch in enumerate(expr):
+            if ch not in cls.RE_FLAGS:
+                break
+            flags |= cls.RE_FLAGS[ch]
+        return flags, expr[idx:] if idx > 0 else expr
+
+    @classmethod
+    def _separate(cls, expr, delim=',', max_split=None, skip_delims=None):
         if not expr:
             return
         counters = {k: 0 for k in _MATCHING_PARENS.values()}
-        start, splits, pos, skipping, delim_len = 0, 0, 0, 0, len(delim) - 1
-        in_quote, escaping = None, False
+        start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
+        in_quote, escaping, skipping = None, False, 0
+        after_op, in_regex_char_group, skip_re = True, False, 0
+
         for idx, char in enumerate(expr):
+            if skip_re > 0:
+                skip_re -= 1
+                continue
             if not in_quote:
                 if char in _MATCHING_PARENS:
                     counters[_MATCHING_PARENS[char]] += 1
                 elif char in counters:
                     counters[char] -= 1
-            if not escaping:
-                if char in _QUOTES and in_quote in (char, None):
-                    in_quote = None if in_quote else char
-                else:
-                    escaping = in_quote and char == '\\'
-            else:
-                escaping = False
+            if not escaping and char in _QUOTES and in_quote in (char, None):
+                if in_quote or after_op or char != '/':
+                    in_quote = None if in_quote and not in_regex_char_group else char
+                    if in_quote is None and char == '/' and delim != '/':
+                        # regexp flags
+                        n_idx = idx + 1
+                        while n_idx < len(expr) and expr[n_idx] in cls.RE_FLAGS:
+                            n_idx += 1
+                        skip_re = n_idx - idx - 1
+                        if skip_re > 0:
+                            continue
+            elif in_quote == '/' and char in '[]':
+                in_regex_char_group = char == '['
+            escaping = not escaping and in_quote and char == '\\'
+            after_op = not in_quote and char in cls.OP_CHARS or (char == ' ' and after_op)
 
             if char != delim[pos] or any(counters.values()) or in_quote:
                 pos = skipping = 0
@@ -313,16 +383,23 @@ class JSInterpreter(object):
             if should_return:
                 return ret, should_return
 
-        m = re.match(r'(?P<var>(?:var|const|let)\s)|return(?:\s+|$)', stmt)
+        m = re.match(r'(?P<var>(?:var|const|let)\s)|return(?:\s+|(?=["\'])|$)|(?P<throw>throw\s+)', stmt)
         if m:
             expr = stmt[len(m.group(0)):].strip()
+            if m.group('throw'):
+                raise JS_Throw(self.interpret_expression(expr, local_vars, allow_recursion))
             should_return = not m.group('var')
         if not expr:
             return None, should_return
 
         if expr[0] in _QUOTES:
             inner, outer = self._separate(expr, expr[0], 1)
-            inner = json.loads(js_to_json(inner + expr[0]))  # , strict=True))
+            if expr[0] == '/':
+                flags, _ = self._regex_flags(outer)
+                inner, outer = inner.replace('"', r'\"'), ''
+                inner = re.compile(js_to_json(inner + expr[0]), flags=flags)  # , strict=True))
+            else:
+                inner = json.loads(js_to_json(inner + expr[0]))  # , strict=True))
             if not outer:
                 return inner, should_return
             expr = self._named_object(local_vars, inner) + outer
@@ -374,22 +451,37 @@ class JSInterpreter(object):
                 for item in self._separate(inner)])
             expr = name + outer
 
-        m = re.match(r'(?P<try>try|finally)\s*|(?:(?P<catch>catch)|(?P<for>for)|(?P<switch>switch))\s*\(', expr)
+        m = re.match(r'''(?x)
+            (?P<try>try|finally)\s*|
+            (?P<catch>catch\s*(?P<err>\(\s*{_NAME_RE}\s*\)))|
+            (?P<switch>switch)\s*\(|
+            (?P<for>for)\s*\(|'''.format(**globals()), expr)
         md = m.groupdict() if m else {}
         if md.get('try'):
             if expr[m.end()] == '{':
                 try_expr, expr = self._separate_at_paren(expr[m.end():], '}')
             else:
                 try_expr, expr = expr[m.end() - 1:], ''
-            ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion)
-            if should_abort:
-                return ret, True
+            try:
+                ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion)
+                if should_abort:
+                    return ret, True
+            except JS_Throw as e:
+                local_vars[self._EXC_NAME] = e.error
+            except Exception as e:
+                # XXX: This works for now, but makes debugging future issues very hard
+                local_vars[self._EXC_NAME] = e
             ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
             return ret, should_abort or should_return
 
         elif md.get('catch'):
-            # We ignore the catch block
-            _, expr = self._separate_at_paren(expr, '}')
+            catch_expr, expr = self._separate_at_paren(expr[m.end():], '}')
+            if self._EXC_NAME in local_vars:
+                catch_vars = local_vars.new_child({m.group('err'): local_vars.pop(self._EXC_NAME)})
+                ret, should_abort = self.interpret_statement(catch_expr, catch_vars, allow_recursion)
+                if should_abort:
+                    return ret, True
+
             ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
             return ret, should_abort or should_return
 
@@ -503,7 +595,7 @@ class JSInterpreter(object):
                 raise self.Exception('List index %s must be integer' % (idx, ), expr=expr)
             idx = int(idx)
             left_val[idx] = self._operator(
-                m.group('op'), left_val[idx], m.group('expr'), expr, local_vars, allow_recursion)
+                m.group('op'), self._index(left_val, idx), m.group('expr'), expr, local_vars, allow_recursion)
             return left_val[idx], should_return
 
         elif expr.isdigit():

From 46b8ae2f520c17aaa756082676788c6287b6809e Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 19 Aug 2022 15:34:33 +0100
Subject: [PATCH 525/860] [jsinterp] Clean up and pull yt-dlp style * add
 compat_re_Pattern * improve compat_collections_chain_map * use class
 JS_Undefined * remove unused code

---
 test/test_jsinterp.py          |  20 +++---
 test/test_youtube_signature.py |   3 +-
 youtube_dl/compat.py           |  21 +++++-
 youtube_dl/jsinterp.py         | 123 ++++++++++++---------------------
 4 files changed, 77 insertions(+), 90 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index faddf00d5..96786a84c 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -11,8 +11,9 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 import math
 import re
 
-from youtube_dl.jsinterp import JSInterpreter
-undefined = JSInterpreter.undefined
+from youtube_dl.compat import compat_re_Pattern
+
+from youtube_dl.jsinterp import JS_Undefined, JSInterpreter
 
 
 class TestJSInterpreter(unittest.TestCase):
@@ -261,12 +262,12 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('''
         function x() { return undefined; }
         ''')
-        self.assertIs(jsi.call_function('x'), undefined)
+        self.assertIs(jsi.call_function('x'), JS_Undefined)
 
         jsi = JSInterpreter('''
         function x() { let v; return v; }
         ''')
-        self.assertIs(jsi.call_function('x'), undefined)
+        self.assertIs(jsi.call_function('x'), JS_Undefined)
 
         jsi = JSInterpreter('''
         function x() { return [undefined === undefined, undefined == undefined, undefined < undefined, undefined > undefined]; }
@@ -307,7 +308,7 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('''
         function x() { let v; return [v>42, v<=42, v&&42, 42&&v]; }
         ''')
-        self.assertEqual(jsi.call_function('x'), [False, False, undefined, undefined])
+        self.assertEqual(jsi.call_function('x'), [False, False, JS_Undefined, JS_Undefined])
 
         jsi = JSInterpreter('function x(){return undefined ?? 42; }')
         self.assertEqual(jsi.call_function('x'), 42)
@@ -326,12 +327,12 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('''
         function x() { let a; return a?.qq; }
         ''')
-        self.assertIs(jsi.call_function('x'), undefined)
+        self.assertIs(jsi.call_function('x'), JS_Undefined)
 
         jsi = JSInterpreter('''
         function x() { let a = {m1: 42, m2: 0 }; return a?.qq; }
         ''')
-        self.assertIs(jsi.call_function('x'), undefined)
+        self.assertIs(jsi.call_function('x'), JS_Undefined)
 
     def test_regex(self):
         jsi = JSInterpreter('''
@@ -342,13 +343,12 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('''
         function x() { let a=/,,[/,913,/](,)}/; return a; }
         ''')
-        # Pythons disagree on the type of a pattern
-        self.assertTrue(isinstance(jsi.call_function('x'), type(re.compile(''))))
+        self.assertIsInstance(jsi.call_function('x'), compat_re_Pattern)
 
         jsi = JSInterpreter('''
         function x() { let a=/,,[/,913,/](,)}/i; return a; }
         ''')
-        self.assertEqual(jsi.call_function('x').flags & re.I, re.I)
+        self.assertEqual(jsi.call_function('x').flags & ~re.U, re.I)
 
 
 if __name__ == '__main__':
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 43e22388d..327d4c40d 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -12,10 +12,11 @@ import io
 import re
 import string
 
+from youtube_dl.compat import compat_str, compat_urlretrieve
+
 from test.helper import FakeYDL
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.jsinterp import JSInterpreter
-from youtube_dl.compat import compat_str, compat_urlretrieve
 
 _SIG_TESTS = [
     (
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 6d2c31a61..3002109ca 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -3023,18 +3023,34 @@ except ImportError:
             self.maps[0].__setitem__(k, v)
             return
 
-        def __delitem__(self, k):
+        def __contains__(self, k):
+            return any((k in m) for m in self.maps)
+
+        def __delitem(self, k):
             if k in self.maps[0]:
                 del self.maps[0][k]
                 return
             raise KeyError(k)
 
+        def __delitem__(self, k):
+            self.__delitem(k)
+
         def __iter__(self):
             return itertools.chain(*reversed(self.maps))
 
         def __len__(self):
             return len(iter(self))
 
+        # to match Py3, don't del directly
+        def pop(self, k, *args):
+            if self.__contains__(k):
+                off = self.__getitem__(k)
+                self.__delitem(k)
+                return off
+            elif len(args) > 0:
+                return args[0]
+            raise KeyError(k)
+
         def new_child(self, m=None, **kwargs):
             m = m or {}
             m.update(kwargs)
@@ -3044,6 +3060,8 @@ except ImportError:
         def parents(self):
             return compat_collections_chain_map(*(self.maps[1:]))
 
+# Pythons disagree on the type of a pattern (RegexObject, _sre.SRE_Pattern, Pattern, ...?)
+compat_re_Pattern = type(re.compile(''))
 
 if sys.version_info < (3, 3):
     def compat_b64decode(s, *args, **kwargs):
@@ -3110,6 +3128,7 @@ __all__ = [
     'compat_os_name',
     'compat_parse_qs',
     'compat_print',
+    'compat_re_Pattern',
     'compat_realpath',
     'compat_setenv',
     'compat_shlex_quote',
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 48c27a1c0..6719d0dfd 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -19,16 +19,12 @@ from .compat import (
     compat_str,
 )
 
-_NAME_RE = r'[a-zA-Z_$][\w$]*'
-
-_UNDEFINED = object()
-
 
 def _js_bit_op(op):
 
     def wrapped(a, b):
         def zeroise(x):
-            return 0 if x in (None, _UNDEFINED) else x
+            return 0 if x in (None, JS_Undefined) else x
         return op(zeroise(a), zeroise(b))
 
     return wrapped
@@ -37,7 +33,7 @@ def _js_bit_op(op):
 def _js_arith_op(op):
 
     def wrapped(a, b):
-        if _UNDEFINED in (a, b):
+        if JS_Undefined in (a, b):
             return float('nan')
         return op(a or 0, b or 0)
 
@@ -45,22 +41,21 @@ def _js_arith_op(op):
 
 
 def _js_div(a, b):
-    if _UNDEFINED in (a, b) or not (a and b):
+    if JS_Undefined in (a, b) or not (a and b):
         return float('nan')
     return float('inf') if not b else operator.truediv(a or 0, b)
 
 
 def _js_mod(a, b):
-    if _UNDEFINED in (a, b) or not b:
+    if JS_Undefined in (a, b) or not b:
         return float('nan')
     return (a or 0) % b
 
 
 def _js_exp(a, b):
     if not b:
-        # even 0 ** 0 !!
-        return 1
-    if _UNDEFINED in (a, b):
+        return 1  # even 0 ** 0 !!
+    elif JS_Undefined in (a, b):
         return float('nan')
     return (a or 0) ** b
 
@@ -68,7 +63,7 @@ def _js_exp(a, b):
 def _js_eq_op(op):
 
     def wrapped(a, b):
-        if set((a, b)) <= set((None, _UNDEFINED)):
+        if set((a, b)) <= set((None, JS_Undefined)):
             return op(a, a)
         return op(a, b)
 
@@ -78,21 +73,28 @@ def _js_eq_op(op):
 def _js_comp_op(op):
 
     def wrapped(a, b):
-        if _UNDEFINED in (a, b):
+        if JS_Undefined in (a, b):
             return False
         return op(a or 0, b or 0)
 
     return wrapped
 
 
+def _js_ternary(cndn, if_true=True, if_false=False):
+    """Simulate JS's ternary operator (cndn?if_true:if_false)"""
+    if cndn in (False, None, 0, '', JS_Undefined):
+        return if_false
+    try:
+        if math.isnan(cndn):  # NB: NaN cannot be checked by membership
+            return if_false
+    except TypeError:
+        pass
+    return if_true
+
+
 # (op, definition) in order of binding priority, tightest first
 # avoid dict to maintain order
 # definition None => Defined in JSInterpreter._operator
-_DOT_OPERATORS = (
-    ('.', None),
-    # TODO: ('?.', None),
-)
-
 _OPERATORS = (
     ('>>', _js_bit_op(operator.rshift)),
     ('<<', _js_bit_op(operator.lshift)),
@@ -130,20 +132,13 @@ _SC_OPERATORS = (
 
 _OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS))
 
+_NAME_RE = r'[a-zA-Z_$][\w$]*'
 _MATCHING_PARENS = dict(zip(*zip('()', '{}', '[]')))
 _QUOTES = '\'"/'
 
 
-def _ternary(cndn, if_true=True, if_false=False):
-    """Simulate JS's ternary operator (cndn?if_true:if_false)"""
-    if cndn in (False, None, 0, '', _UNDEFINED):
-        return if_false
-    try:
-        if math.isnan(cndn):  # NB: NaN cannot be checked by membership
-            return if_false
-    except TypeError:
-        pass
-    return if_true
+class JS_Undefined(object):
+    pass
 
 
 class JS_Break(ExtractorError):
@@ -167,7 +162,7 @@ class LocalNameSpace(ChainMap):
         try:
             return super(LocalNameSpace, self).__getitem__(key)
         except KeyError:
-            return _UNDEFINED
+            return JS_Undefined
 
     def __setitem__(self, key, value):
         for scope in self.maps:
@@ -179,24 +174,6 @@ class LocalNameSpace(ChainMap):
     def __delitem__(self, key):
         raise NotImplementedError('Deleting is not supported')
 
-    # except
-    def pop(self, key, *args):
-        try:
-            off = self.__getitem__(key)
-            super(LocalNameSpace, self).__delitem__(key)
-            return off
-        except KeyError:
-            if len(args) > 0:
-                return args[0]
-            raise
-
-    def __contains__(self, key):
-        try:
-            super(LocalNameSpace, self).__getitem__(key)
-            return True
-        except KeyError:
-            return False
-
     def __repr__(self):
         return 'LocalNameSpace%s' % (self.maps, )
 
@@ -204,9 +181,7 @@ class LocalNameSpace(ChainMap):
 class JSInterpreter(object):
     __named_object_counter = 0
 
-    undefined = _UNDEFINED
-
-    RE_FLAGS = {
+    _RE_FLAGS = {
         # special knowledge: Python's re flags are bitmask values, current max 128
         # invent new bitmask values well above that for literal parsing
         # TODO: new pattern class to execute matches with these flags
@@ -257,10 +232,10 @@ class JSInterpreter(object):
         if not expr:
             return flags, expr
         for idx, ch in enumerate(expr):
-            if ch not in cls.RE_FLAGS:
+            if ch not in cls._RE_FLAGS:
                 break
-            flags |= cls.RE_FLAGS[ch]
-        return flags, expr[idx:] if idx > 0 else expr
+            flags |= cls._RE_FLAGS[ch]
+        return flags, expr[idx + 1:]
 
     @classmethod
     def _separate(cls, expr, delim=',', max_split=None, skip_delims=None):
@@ -283,14 +258,6 @@ class JSInterpreter(object):
             if not escaping and char in _QUOTES and in_quote in (char, None):
                 if in_quote or after_op or char != '/':
                     in_quote = None if in_quote and not in_regex_char_group else char
-                    if in_quote is None and char == '/' and delim != '/':
-                        # regexp flags
-                        n_idx = idx + 1
-                        while n_idx < len(expr) and expr[n_idx] in cls.RE_FLAGS:
-                            n_idx += 1
-                        skip_re = n_idx - idx - 1
-                        if skip_re > 0:
-                            continue
             elif in_quote == '/' and char in '[]':
                 in_regex_char_group = char == '['
             escaping = not escaping and in_quote and char == '\\'
@@ -336,13 +303,13 @@ class JSInterpreter(object):
 
     def _operator(self, op, left_val, right_expr, expr, local_vars, allow_recursion):
         if op in ('||', '&&'):
-            if (op == '&&') ^ _ternary(left_val):
+            if (op == '&&') ^ _js_ternary(left_val):
                 return left_val  # short circuiting
         elif op == '??':
-            if left_val not in (None, self.undefined):
+            if left_val not in (None, JS_Undefined):
                 return left_val
         elif op == '?':
-            right_expr = _ternary(left_val, *self._separate(right_expr, ':', 1))
+            right_expr = _js_ternary(left_val, *self._separate(right_expr, ':', 1))
 
         right_val = self.interpret_expression(right_expr, local_vars, allow_recursion)
         opfunc = op and next((v for k, v in self._all_operators() if k == op), None)
@@ -361,7 +328,7 @@ class JSInterpreter(object):
             return obj[int(idx)] if isinstance(obj, list) else obj[idx]
         except Exception as e:
             if allow_undefined:
-                return self.undefined
+                return JS_Undefined
             raise self.Exception('Cannot get index {idx}'.format(**locals()), expr=repr(obj), cause=e)
 
     def _dump(self, obj, namespace):
@@ -395,9 +362,8 @@ class JSInterpreter(object):
         if expr[0] in _QUOTES:
             inner, outer = self._separate(expr, expr[0], 1)
             if expr[0] == '/':
-                flags, _ = self._regex_flags(outer)
-                inner, outer = inner.replace('"', r'\"'), ''
-                inner = re.compile(js_to_json(inner + expr[0]), flags=flags)  # , strict=True))
+                flags, outer = self._regex_flags(outer)
+                inner = re.compile(inner[1:], flags=flags)  # , strict=True))
             else:
                 inner = json.loads(js_to_json(inner + expr[0]))  # , strict=True))
             if not outer:
@@ -422,7 +388,7 @@ class JSInterpreter(object):
 
         if expr.startswith('{'):
             inner, outer = self._separate_at_paren(expr, '}')
-            # try for object expression
+            # try for object expression (Map)
             sub_expressions = [list(self._separate(sub_expr.strip(), ':', 1)) for sub_expr in self._separate(inner)]
             if all(len(sub_expr) == 2 for sub_expr in sub_expressions):
                 return dict(
@@ -455,7 +421,8 @@ class JSInterpreter(object):
             (?P<try>try|finally)\s*|
             (?P<catch>catch\s*(?P<err>\(\s*{_NAME_RE}\s*\)))|
             (?P<switch>switch)\s*\(|
-            (?P<for>for)\s*\(|'''.format(**globals()), expr)
+            (?P<for>for)\s*\(|
+            '''.format(**globals()), expr)
         md = m.groupdict() if m else {}
         if md.get('try'):
             if expr[m.end()] == '{':
@@ -500,7 +467,7 @@ class JSInterpreter(object):
             start, cndn, increment = self._separate(constructor, ';')
             self.interpret_expression(start, local_vars, allow_recursion)
             while True:
-                if not _ternary(self.interpret_expression(cndn, local_vars, allow_recursion)):
+                if not _js_ternary(self.interpret_expression(cndn, local_vars, allow_recursion)):
                     break
                 try:
                     ret, should_abort = self.interpret_statement(body, local_vars, allow_recursion)
@@ -587,7 +554,7 @@ class JSInterpreter(object):
                 local_vars[m.group('out')] = self._operator(
                     m.group('op'), left_val, m.group('expr'), expr, local_vars, allow_recursion)
                 return local_vars[m.group('out')], should_return
-            elif left_val in (None, self.undefined):
+            elif left_val in (None, JS_Undefined):
                 raise self.Exception('Cannot index undefined variable ' + m.group('out'), expr=expr)
 
             idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
@@ -607,7 +574,7 @@ class JSInterpreter(object):
             raise JS_Continue()
 
         elif expr == 'undefined':
-            return self.undefined, should_return
+            return JS_Undefined, should_return
 
         elif md.get('return'):
             return local_vars[m.group('name')], should_return
@@ -663,9 +630,9 @@ class JSInterpreter(object):
                     'Math': float,
                 }
                 obj = local_vars.get(variable)
-                if obj in (self.undefined, None):
-                    obj = types.get(variable, self.undefined)
-                if obj is self.undefined:
+                if obj in (JS_Undefined, None):
+                    obj = types.get(variable, JS_Undefined)
+                if obj is JS_Undefined:
                     try:
                         if variable not in self._objects:
                             self._objects[variable] = self.extract_object(variable)
@@ -674,8 +641,8 @@ class JSInterpreter(object):
                         if not nullish:
                             raise
 
-                if nullish and obj is self.undefined:
-                    return self.undefined
+                if nullish and obj is JS_Undefined:
+                    return JS_Undefined
 
                 # Member access
                 if arg_str is None:

From fd3f3bebd0699f4b782a24a503093c965c4f4f5e Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 19 Aug 2022 19:11:08 +0100
Subject: [PATCH 526/860] [uktvplay] Support domain without .uktv

---
 youtube_dl/extractor/uktvplay.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/uktvplay.py b/youtube_dl/extractor/uktvplay.py
index f28fd514d..9ef9638cd 100644
--- a/youtube_dl/extractor/uktvplay.py
+++ b/youtube_dl/extractor/uktvplay.py
@@ -5,7 +5,7 @@ from .common import InfoExtractor
 
 
 class UKTVPlayIE(InfoExtractor):
-    _VALID_URL = r'https?://uktvplay\.uktv\.co\.uk/(?:.+?\?.*?\bvideo=|([^/]+/)*watch-online/)(?P<id>\d+)'
+    _VALID_URL = r'https?://uktvplay\.(?:uktv\.)?co\.uk/(?:.+?\?.*?\bvideo=|([^/]+/)*watch-online/)(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://uktvplay.uktv.co.uk/shows/world-at-war/c/200/watch-online/?video=2117008346001',
         'info_dict': {

From a8d5316aaf3dc740aad486b8c394b2f3e70f5a58 Mon Sep 17 00:00:00 2001
From: gudata <gudata@users.noreply.github.com>
Date: Fri, 19 Aug 2022 23:00:21 +0300
Subject: [PATCH 527/860] [infoq] Avoid crash if the page has no `mp3Form`

* proposed fix for issue #31131, aligns with yt-dlp

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/infoq.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/infoq.py b/youtube_dl/extractor/infoq.py
index 0a70a1fb4..60b02b699 100644
--- a/youtube_dl/extractor/infoq.py
+++ b/youtube_dl/extractor/infoq.py
@@ -1,6 +1,9 @@
 # coding: utf-8
 
 from __future__ import unicode_literals
+from ..utils import (
+    ExtractorError,
+)
 
 from ..compat import (
     compat_b64decode,
@@ -90,7 +93,11 @@ class InfoQIE(BokeCCBaseIE):
         }]
 
     def _extract_http_audio(self, webpage, video_id):
-        fields = self._form_hidden_inputs('mp3Form', webpage)
+        try:
+            fields = self._form_hidden_inputs('mp3Form', webpage)
+        except ExtractorError:
+            fields = {}
+
         http_audio_url = fields.get('filename')
         if not http_audio_url:
             return []

From 556862bc911bb54435b7b0b01451789b884b0390 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 21 Aug 2022 00:19:19 +0100
Subject: [PATCH 528/860] [utils] Ensure RFC3986 encoding result is unicode

---
 youtube_dl/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index a5f584ec5..fea38ed32 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3970,7 +3970,8 @@ def escape_rfc3986(s):
     """Escape non-ASCII characters as suggested by RFC 3986"""
     if sys.version_info < (3, 0) and isinstance(s, compat_str):
         s = s.encode('utf-8')
-    return compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")
+    # ensure unicode: after quoting, it can always be converted
+    return compat_str(compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]"))
 
 
 def escape_url(url):

From 66e58dccc29de65cc95ee97915987d785b2b4b31 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 21 Aug 2022 00:21:02 +0100
Subject: [PATCH 529/860] [core] Avoid processing empty format list after
 removing bad formats * also ensure compat encoding of error strings

---
 youtube_dl/YoutubeDL.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index e77b8d50c..8e8546596 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -721,7 +721,7 @@ class YoutubeDL(object):
                 filename = encodeFilename(filename, True).decode(preferredencoding())
             return sanitize_path(filename)
         except ValueError as err:
-            self.report_error('Error in output template: ' + str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
+            self.report_error('Error in output template: ' + error_to_compat_str(err) + ' (encoding: ' + repr(preferredencoding()) + ')')
             return None
 
     def _match_entry(self, info_dict, incomplete):
@@ -1570,9 +1570,6 @@ class YoutubeDL(object):
         else:
             formats = info_dict['formats']
 
-        if not formats:
-            raise ExtractorError('No video formats found!')
-
         def is_wellformed(f):
             url = f.get('url')
             if not url:
@@ -1585,7 +1582,10 @@ class YoutubeDL(object):
             return True
 
         # Filter out malformed formats for better extraction robustness
-        formats = list(filter(is_wellformed, formats))
+        formats = list(filter(is_wellformed, formats or []))
+
+        if not formats:
+            raise ExtractorError('No video formats found!')
 
         formats_dict = {}
 
@@ -2058,7 +2058,7 @@ class YoutubeDL(object):
                 try:
                     self.post_process(filename, info_dict)
                 except (PostProcessingError) as err:
-                    self.report_error('postprocessing: %s' % str(err))
+                    self.report_error('postprocessing: %s' % error_to_compat_str(err))
                     return
                 self.record_download_archive(info_dict)
                 # avoid possible nugatory search for further items (PR #26638)

From 573b13410e5c2f939676116e2700ec8efd9cf97b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 25 Aug 2022 12:14:59 +0100
Subject: [PATCH 530/860] [YouTube] Improve error check for n-sig processing

---
 youtube_dl/extractor/youtube.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 91a3b6058..3d12e2e4a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1500,7 +1500,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         return lambda s: jsi.extract_function_from_code(*func_code)([s])
 
     def _n_descramble(self, n_param, player_url, video_id):
-        """Compute the response to YT's "n" parameter challenge
+        """Compute the response to YT's "n" parameter challenge,
+           or None
 
         Args:
         n_param     -- challenge string that is the value of the
@@ -1518,7 +1519,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if player_id not in self._player_cache:
                 self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
             func = self._player_cache[player_id]
-            self._player_cache[sig_id] = func(n_param)
+            ret = func(n_param)
+            if ret.startswith('enhanced_except_'):
+                raise ExtractorError('Unhandled exception in decode')
+            self._player_cache[sig_id] = ret
             if self._downloader.params.get('verbose', False):
                 self._downloader.to_screen('[debug] [%s] %s' % (self.IE_NAME, 'Decrypted nsig {0} => {1}'.format(n_param, self._player_cache[sig_id])))
             return self._player_cache[sig_id]
@@ -1539,10 +1543,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 continue
             n_param = n_param[-1]
             n_response = self._n_descramble(n_param, player_url, video_id)
-            if n_response:
-                qs['n'] = [n_response]
-                fmt['url'] = compat_urlparse.urlunparse(
-                    parsed_fmt_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
+            if n_response is None:
+                # give up if descrambling failed
+                break
+            qs['n'] = [n_response]
+            fmt['url'] = compat_urlparse.urlunparse(
+                parsed_fmt_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
 
     def _mark_watched(self, video_id, player_response):
         playback_url = url_or_none(try_get(

From d619dd712f63aab1964f8fdde9ceea514a5e581d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 25 Aug 2022 12:16:10 +0100
Subject: [PATCH 531/860] [jsinterp] Fix bug in operator precedence * from
 https://github.com/yt-dlp/yt-dlp/commit/164b03c4864b0d44cfee5e7702f7c2317164a6cf
 * added tests

---
 test/test_jsinterp.py          | 25 +++++++++++++++++++++++++
 test/test_youtube_signature.py |  4 ++++
 youtube_dl/jsinterp.py         |  7 ++++++-
 3 files changed, 35 insertions(+), 1 deletion(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 96786a84c..0a97bdbc4 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -192,6 +192,31 @@ class TestJSInterpreter(unittest.TestCase):
         ''')
         self.assertEqual(jsi.call_function('x'), 10)
 
+    def test_catch(self):
+        jsi = JSInterpreter('''
+        function x() { try{throw 10} catch(e){return 5} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 5)
+
+    @unittest.expectedFailure
+    def test_finally(self):
+        jsi = JSInterpreter('''
+        function x() { try{throw 10} finally {return 42} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 42)
+        jsi = JSInterpreter('''
+        function x() { try{throw 10} catch(e){return 5} finally {return 42} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 42)
+
+    def test_nested_try(self):
+        jsi = JSInterpreter('''
+        function x() {try {
+            try{throw 10} finally {throw 42} 
+            } catch(e){return 5} }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 5)
+
     def test_for_loop_continue(self):
         jsi = JSInterpreter('''
         function x() { a=0; for (i=0; i-10; i++) { continue; a++ } return a }
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 327d4c40d..4bb0a30b0 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -111,6 +111,10 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/1f7d5369/player_ias.vflset/en_US/base.js',
         'batNX7sYqIJdkJ', 'IhOkL_zxbkOZBw',
     ),
+    (
+        'https://www.youtube.com/s/player/dc0c6770/player_ias.vflset/en_US/base.js',
+        '5EHDMgYLV6HPGk_Mu-kk', 'n9lUJLHbxUI0GQ',
+    ),
 ]
 
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 6719d0dfd..a8456ec1c 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -5,6 +5,7 @@ import json
 import math
 import operator
 import re
+from collections import Counter
 
 from .utils import (
     error_to_compat_str,
@@ -108,8 +109,8 @@ _OPERATORS = (
 
 _COMP_OPERATORS = (
     ('===', operator.is_),
-    ('==', _js_eq_op(operator.eq)),
     ('!==', operator.is_not),
+    ('==', _js_eq_op(operator.eq)),
     ('!=', _js_eq_op(operator.ne)),
     ('<=', _js_comp_op(operator.le)),
     ('>=', _js_comp_op(operator.ge)),
@@ -241,7 +242,9 @@ class JSInterpreter(object):
     def _separate(cls, expr, delim=',', max_split=None, skip_delims=None):
         if not expr:
             return
+        # collections.Counter() is ~10% slower
         counters = {k: 0 for k in _MATCHING_PARENS.values()}
+        # counters = Counter()
         start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
         in_quote, escaping, skipping = None, False, 0
         after_op, in_regex_char_group, skip_re = True, False, 0
@@ -442,6 +445,7 @@ class JSInterpreter(object):
             return ret, should_abort or should_return
 
         elif md.get('catch'):
+
             catch_expr, expr = self._separate_at_paren(expr[m.end():], '}')
             if self._EXC_NAME in local_vars:
                 catch_vars = local_vars.new_child({m.group('err'): local_vars.pop(self._EXC_NAME)})
@@ -450,6 +454,7 @@ class JSInterpreter(object):
                     return ret, True
 
             ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
+
             return ret, should_abort or should_return
 
         elif md.get('for'):

From 4c6fba37650d60acbd32a9f2d6e2468a730d0f1c Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 26 Aug 2022 08:17:54 +0100
Subject: [PATCH 532/860] [jsinterp] Improve try/catch/finally support

---
 test/test_jsinterp.py  | 14 ++++++-
 youtube_dl/jsinterp.py | 88 +++++++++++++++++++++++-------------------
 2 files changed, 61 insertions(+), 41 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 0a97bdbc4..fb4882d00 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -74,6 +74,9 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('function f(){return 0 ?? 42;}')
         self.assertEqual(jsi.call_function('f'), 0)
 
+        jsi = JSInterpreter('function f(){return "life, the universe and everything" < 42;}')
+        self.assertFalse(jsi.call_function('f'))
+
     def test_array_access(self):
         jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}')
         self.assertEqual(jsi.call_function('f'), [5, 2, 7])
@@ -198,7 +201,6 @@ class TestJSInterpreter(unittest.TestCase):
         ''')
         self.assertEqual(jsi.call_function('x'), 5)
 
-    @unittest.expectedFailure
     def test_finally(self):
         jsi = JSInterpreter('''
         function x() { try{throw 10} finally {return 42} }
@@ -212,7 +214,7 @@ class TestJSInterpreter(unittest.TestCase):
     def test_nested_try(self):
         jsi = JSInterpreter('''
         function x() {try {
-            try{throw 10} finally {throw 42} 
+            try{throw 10} finally {throw 42}
             } catch(e){return 5} }
         ''')
         self.assertEqual(jsi.call_function('x'), 5)
@@ -229,6 +231,14 @@ class TestJSInterpreter(unittest.TestCase):
         ''')
         self.assertEqual(jsi.call_function('x'), 0)
 
+    def test_for_loop_try(self):
+        jsi = JSInterpreter('''
+        function x() {
+            for (i=0; i-10; i++) { try { if (i == 5) throw i} catch {return 10} finally {break} };
+            return 42 }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 42)
+
     def test_literal_list(self):
         jsi = JSInterpreter('''
         function x() { return [1, 2, "asdf", [5, 6, 7]][3] }
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index a8456ec1c..08726e478 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -5,7 +5,6 @@ import json
 import math
 import operator
 import re
-from collections import Counter
 
 from .utils import (
     error_to_compat_str,
@@ -15,6 +14,7 @@ from .utils import (
     unified_timestamp,
 )
 from .compat import (
+    compat_basestring,
     compat_collections_chain_map as ChainMap,
     compat_itertools_zip_longest as zip_longest,
     compat_str,
@@ -76,6 +76,10 @@ def _js_comp_op(op):
     def wrapped(a, b):
         if JS_Undefined in (a, b):
             return False
+        if isinstance(a, compat_basestring):
+            b = compat_str(b or 0)
+        elif isinstance(b, compat_basestring):
+            a = compat_str(a or 0)
         return op(a or 0, b or 0)
 
     return wrapped
@@ -195,7 +199,6 @@ class JSInterpreter(object):
         'y': 4096,  # Perform a "sticky" search that matches starting at the current position in the target string
     }
 
-    _EXC_NAME = '__youtube_dl_exception__'
     _OBJ_NAME = '__youtube_dl_jsinterp_obj'
 
     OP_CHARS = None
@@ -242,9 +245,8 @@ class JSInterpreter(object):
     def _separate(cls, expr, delim=',', max_split=None, skip_delims=None):
         if not expr:
             return
-        # collections.Counter() is ~10% slower
+        # collections.Counter() is ~10% slower in both 2.7 and 3.9
         counters = {k: 0 for k in _MATCHING_PARENS.values()}
-        # counters = Counter()
         start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
         in_quote, escaping, skipping = None, False, 0
         after_op, in_regex_char_group, skip_re = True, False, 0
@@ -291,7 +293,9 @@ class JSInterpreter(object):
         yield expr[start:]
 
     @classmethod
-    def _separate_at_paren(cls, expr, delim):
+    def _separate_at_paren(cls, expr, delim=None):
+        if delim is None:
+            delim = expr and _MATCHING_PARENS[expr[0]]
         separated = list(cls._separate(expr, delim, 1))
 
         if len(separated) < 2:
@@ -376,7 +380,7 @@ class JSInterpreter(object):
         if expr.startswith('new '):
             obj = expr[4:]
             if obj.startswith('Date('):
-                left, right = self._separate_at_paren(obj[4:], ')')
+                left, right = self._separate_at_paren(obj[4:])
                 expr = unified_timestamp(
                     self.interpret_expression(left, local_vars, allow_recursion), False)
                 if not expr:
@@ -390,7 +394,7 @@ class JSInterpreter(object):
             return None, should_return
 
         if expr.startswith('{'):
-            inner, outer = self._separate_at_paren(expr, '}')
+            inner, outer = self._separate_at_paren(expr)
             # try for object expression (Map)
             sub_expressions = [list(self._separate(sub_expr.strip(), ':', 1)) for sub_expr in self._separate(inner)]
             if all(len(sub_expr) == 2 for sub_expr in sub_expressions):
@@ -406,7 +410,7 @@ class JSInterpreter(object):
                 expr = self._dump(inner, local_vars) + outer
 
         if expr.startswith('('):
-            inner, outer = self._separate_at_paren(expr, ')')
+            inner, outer = self._separate_at_paren(expr)
             inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion)
             if not outer or should_abort:
                 return inner, should_abort or should_return
@@ -414,57 +418,63 @@ class JSInterpreter(object):
                 expr = self._dump(inner, local_vars) + outer
 
         if expr.startswith('['):
-            inner, outer = self._separate_at_paren(expr, ']')
+            inner, outer = self._separate_at_paren(expr)
             name = self._named_object(local_vars, [
                 self.interpret_expression(item, local_vars, allow_recursion)
                 for item in self._separate(inner)])
             expr = name + outer
 
         m = re.match(r'''(?x)
-            (?P<try>try|finally)\s*|
-            (?P<catch>catch\s*(?P<err>\(\s*{_NAME_RE}\s*\)))|
-            (?P<switch>switch)\s*\(|
-            (?P<for>for)\s*\(|
-            '''.format(**globals()), expr)
+                (?P<try>try)\s*\{|
+                (?P<switch>switch)\s*\(|
+                (?P<for>for)\s*\(
+                ''', expr)
         md = m.groupdict() if m else {}
         if md.get('try'):
-            if expr[m.end()] == '{':
-                try_expr, expr = self._separate_at_paren(expr[m.end():], '}')
-            else:
-                try_expr, expr = expr[m.end() - 1:], ''
+            try_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
+            err = None
             try:
                 ret, should_abort = self.interpret_statement(try_expr, local_vars, allow_recursion)
                 if should_abort:
                     return ret, True
-            except JS_Throw as e:
-                local_vars[self._EXC_NAME] = e.error
             except Exception as e:
                 # XXX: This works for now, but makes debugging future issues very hard
-                local_vars[self._EXC_NAME] = e
-            ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
-            return ret, should_abort or should_return
+                err = e
 
-        elif md.get('catch'):
+            pending = (None, False)
+            m = re.match(r'catch\s*(?P<err>\(\s*{_NAME_RE}\s*\))?\{{'.format(**globals()), expr)
+            if m:
+                sub_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
+                if err:
+                    catch_vars = {}
+                    if m.group('err'):
+                        catch_vars[m.group('err')] = err.error if isinstance(err, JS_Throw) else err
+                    catch_vars = local_vars.new_child(m=catch_vars)
+                    err = None
+                    pending = self.interpret_statement(sub_expr, catch_vars, allow_recursion)
 
-            catch_expr, expr = self._separate_at_paren(expr[m.end():], '}')
-            if self._EXC_NAME in local_vars:
-                catch_vars = local_vars.new_child({m.group('err'): local_vars.pop(self._EXC_NAME)})
-                ret, should_abort = self.interpret_statement(catch_expr, catch_vars, allow_recursion)
+            m = re.match(r'finally\s*\{', expr)
+            if m:
+                sub_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
+                ret, should_abort = self.interpret_statement(sub_expr, local_vars, allow_recursion)
                 if should_abort:
                     return ret, True
 
-            ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
+            ret, should_abort = pending
+            if should_abort:
+                return ret, True
 
-            return ret, should_abort or should_return
+            if err:
+                raise err
 
         elif md.get('for'):
-            constructor, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
+            constructor, remaining = self._separate_at_paren(expr[m.end() - 1:])
             if remaining.startswith('{'):
-                body, expr = self._separate_at_paren(remaining, '}')
+                body, expr = self._separate_at_paren(remaining)
             else:
                 switch_m = re.match(r'switch\s*\(', remaining)  # FIXME
                 if switch_m:
-                    switch_val, remaining = self._separate_at_paren(remaining[switch_m.end() - 1:], ')')
+                    switch_val, remaining = self._separate_at_paren(remaining[switch_m.end() - 1:])
                     body, expr = self._separate_at_paren(remaining, '}')
                     body = 'switch(%s){%s}' % (switch_val, body)
                 else:
@@ -483,11 +493,9 @@ class JSInterpreter(object):
                 except JS_Continue:
                     pass
                 self.interpret_expression(increment, local_vars, allow_recursion)
-            ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
-            return ret, should_abort or should_return
 
         elif md.get('switch'):
-            switch_val, remaining = self._separate_at_paren(expr[m.end() - 1:], ')')
+            switch_val, remaining = self._separate_at_paren(expr[m.end() - 1:])
             switch_val = self.interpret_expression(switch_val, local_vars, allow_recursion)
             body, expr = self._separate_at_paren(remaining, '}')
             items = body.replace('default:', 'case default:').split('case ')[1:]
@@ -510,6 +518,8 @@ class JSInterpreter(object):
                         break
                 if matched:
                     break
+
+        if md:
             ret, should_abort = self.interpret_statement(expr, local_vars, allow_recursion)
             return ret, should_abort or should_return
 
@@ -618,7 +628,7 @@ class JSInterpreter(object):
                 member = self.interpret_expression(m.group('member2'), local_vars, allow_recursion)
             arg_str = expr[m.end():]
             if arg_str.startswith('('):
-                arg_str, remaining = self._separate_at_paren(arg_str, ')')
+                arg_str, remaining = self._separate_at_paren(arg_str)
             else:
                 arg_str, remaining = None, arg_str
 
@@ -795,7 +805,7 @@ class JSInterpreter(object):
                 \((?P<args>[^)]*)\)\s*
                 (?P<code>{.+})''' % {'name': re.escape(funcname)},
             self.code)
-        code, _ = self._separate_at_paren(func_m.group('code'), '}')  # refine the match
+        code, _ = self._separate_at_paren(func_m.group('code'))  # refine the match
         if func_m is None:
             raise self.Exception('Could not find JS function "{funcname}"'.format(**locals()))
         return self.build_arglist(func_m.group('args')), code
@@ -810,7 +820,7 @@ class JSInterpreter(object):
             if mobj is None:
                 break
             start, body_start = mobj.span()
-            body, remaining = self._separate_at_paren(code[body_start - 1:], '}')
+            body, remaining = self._separate_at_paren(code[body_start - 1:])
             name = self._named_object(
                 local_vars,
                 self.extract_function_from_code(

From 0f6422590e44e99e9b81cf2367666efe89fae3aa Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 26 Aug 2022 10:17:56 +0100
Subject: [PATCH 533/860] [compat] Replace deficient ChainMap class in Py3.3
 and earlier

---
 youtube_dl/compat.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 3002109ca..366a93924 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -3004,8 +3004,11 @@ except ImportError:
 # new class in collections
 try:
     from collections import ChainMap as compat_collections_chain_map
+    # Py3.3's ChainMap is deficient
+    if sys.version_info <= (3, 3):
+        raise ImportError
 except ImportError:
-    # Py < 3.3
+    # Py <= 3.3
     class compat_collections_chain_map(compat_collections_abc.MutableMapping):
 
         maps = [{}]
@@ -3060,6 +3063,7 @@ except ImportError:
         def parents(self):
             return compat_collections_chain_map(*(self.maps[1:]))
 
+
 # Pythons disagree on the type of a pattern (RegexObject, _sre.SRE_Pattern, Pattern, ...?)
 compat_re_Pattern = type(re.compile(''))
 

From ed5c44e7b74ac77f87ca5ed6cb5e964a0c6a0678 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 26 Aug 2022 12:22:01 +0100
Subject: [PATCH 534/860] [compat] Replace deficient ChainMap class in Py3.3
 and earlier * fix version check

---
 youtube_dl/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 366a93924..eca6d63de 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -3005,7 +3005,7 @@ except ImportError:
 try:
     from collections import ChainMap as compat_collections_chain_map
     # Py3.3's ChainMap is deficient
-    if sys.version_info <= (3, 3):
+    if sys.version_info < (3, 4):
         raise ImportError
 except ImportError:
     # Py <= 3.3

From 4050e10a4c3445c5399239567eb074acb2f65c18 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 29 Aug 2022 13:02:17 +0100
Subject: [PATCH 535/860] [options] Document that postprocessing is not forced
 by --postprocessor-args

Resolves #30307
---
 youtube_dl/options.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index f6621ef91..f6d2b0898 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -801,7 +801,7 @@ def parseOpts(overrideArguments=None):
     postproc.add_option(
         '--postprocessor-args',
         dest='postprocessor_args', metavar='ARGS',
-        help='Give these arguments to the postprocessor')
+        help='Give these arguments to the postprocessor (if postprocessing is required)')
     postproc.add_option(
         '-k', '--keep-video',
         action='store_true', dest='keepvideo', default=False,

From 55c823634db890a328ffc23588fcd6f35d9b3ddf Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 31 Aug 2022 23:22:48 +0100
Subject: [PATCH 536/860] [jsinterp] Handle new YT players 113ca41c, c57c113c *
 add NaN * allow any white-space character for `after_op` * align with yt-dlp
 f26af78a8ac11d9d617ed31ea5282cfaa5bcbcfa (charcodeAt and bitwise overflow) *
 allow escaping in regex, fixing player c57c113c

---
 test/test_jsinterp.py          | 21 ++++++++++++++++
 test/test_youtube_signature.py | 16 ++++++++++++
 youtube_dl/jsinterp.py         | 46 +++++++++++++++++++++-------------
 3 files changed, 65 insertions(+), 18 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index fb4882d00..5121c8cf8 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -135,6 +135,11 @@ class TestJSInterpreter(unittest.TestCase):
         self.assertEqual(jsi.call_function('x'), [20, 20, 30, 40, 50])
 
     def test_builtins(self):
+        jsi = JSInterpreter('''
+        function x() { return NaN }
+        ''')
+        self.assertTrue(math.isnan(jsi.call_function('x')))
+
         jsi = JSInterpreter('''
         function x() { return new Date('Wednesday 31 December 1969 18:01:26 MDT') - 0; }
         ''')
@@ -385,6 +390,22 @@ class TestJSInterpreter(unittest.TestCase):
         ''')
         self.assertEqual(jsi.call_function('x').flags & ~re.U, re.I)
 
+    def test_char_code_at(self):
+        jsi = JSInterpreter('function x(i){return "test".charCodeAt(i)}')
+        self.assertEqual(jsi.call_function('x', 0), 116)
+        self.assertEqual(jsi.call_function('x', 1), 101)
+        self.assertEqual(jsi.call_function('x', 2), 115)
+        self.assertEqual(jsi.call_function('x', 3), 116)
+        self.assertEqual(jsi.call_function('x', 4), None)
+        self.assertEqual(jsi.call_function('x', 'not_a_number'), 116)
+
+    def test_bitwise_operators_overflow(self):
+        jsi = JSInterpreter('function x(){return -524999584 << 5}')
+        self.assertEqual(jsi.call_function('x'), 379882496)
+
+        jsi = JSInterpreter('function x(){return 1236566549 << 5}')
+        self.assertEqual(jsi.call_function('x'), 915423904)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 4bb0a30b0..ec914a871 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -111,10 +111,26 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/1f7d5369/player_ias.vflset/en_US/base.js',
         'batNX7sYqIJdkJ', 'IhOkL_zxbkOZBw',
     ),
+    (
+        'https://www.youtube.com/s/player/009f1d77/player_ias.vflset/en_US/base.js',
+        '5dwFHw8aFWQUQtffRq', 'audescmLUzI3jw',
+    ),
     (
         'https://www.youtube.com/s/player/dc0c6770/player_ias.vflset/en_US/base.js',
         '5EHDMgYLV6HPGk_Mu-kk', 'n9lUJLHbxUI0GQ',
     ),
+    (
+        'https://www.youtube.com/s/player/c2199353/player_ias.vflset/en_US/base.js',
+        '5EHDMgYLV6HPGk_Mu-kk', 'AD5rgS85EkrE7',
+    ),
+    (
+        'https://www.youtube.com/s/player/113ca41c/player_ias.vflset/en_US/base.js',
+        'cgYl-tlYkhjT7A', 'hI7BBr2zUgcmMg',
+    ),
+    (
+        'https://www.youtube.com/s/player/c57c113c/player_ias.vflset/en_US/base.js',
+        '-Txvy6bT5R6LqgnQNx', 'dcklJCnRUHbgSg',
+    ),
 ]
 
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 08726e478..d13329396 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -23,10 +23,11 @@ from .compat import (
 
 def _js_bit_op(op):
 
+    def zeroise(x):
+        return 0 if x in (None, JS_Undefined) else x
+
     def wrapped(a, b):
-        def zeroise(x):
-            return 0 if x in (None, JS_Undefined) else x
-        return op(zeroise(a), zeroise(b))
+        return op(zeroise(a), zeroise(b)) & 0xffffffff
 
     return wrapped
 
@@ -44,7 +45,7 @@ def _js_arith_op(op):
 def _js_div(a, b):
     if JS_Undefined in (a, b) or not (a and b):
         return float('nan')
-    return float('inf') if not b else operator.truediv(a or 0, b)
+    return operator.truediv(a or 0, b) if b else float('inf')
 
 
 def _js_mod(a, b):
@@ -260,13 +261,14 @@ class JSInterpreter(object):
                     counters[_MATCHING_PARENS[char]] += 1
                 elif char in counters:
                     counters[char] -= 1
-            if not escaping and char in _QUOTES and in_quote in (char, None):
-                if in_quote or after_op or char != '/':
-                    in_quote = None if in_quote and not in_regex_char_group else char
-            elif in_quote == '/' and char in '[]':
-                in_regex_char_group = char == '['
+            if not escaping:
+                if char in _QUOTES and in_quote in (char, None):
+                    if in_quote or after_op or char != '/':
+                        in_quote = None if in_quote and not in_regex_char_group else char
+                elif in_quote == '/' and char in '[]':
+                    in_regex_char_group = char == '['
             escaping = not escaping and in_quote and char == '\\'
-            after_op = not in_quote and char in cls.OP_CHARS or (char == ' ' and after_op)
+            after_op = not in_quote and (char in cls.OP_CHARS or (char.isspace() and after_op))
 
             if char != delim[pos] or any(counters.values()) or in_quote:
                 pos = skipping = 0
@@ -590,6 +592,8 @@ class JSInterpreter(object):
 
         elif expr == 'undefined':
             return JS_Undefined, should_return
+        elif expr == 'NaN':
+            return float('NaN'), should_return
 
         elif md.get('return'):
             return local_vars[m.group('name')], should_return
@@ -635,7 +639,8 @@ class JSInterpreter(object):
             def assertion(cndn, msg):
                 """ assert, but without risk of getting optimized out """
                 if not cndn:
-                    raise ExtractorError('{member} {msg}'.format(**locals()), expr=expr)
+                    memb = member
+                    raise self.Exception('{member} {msg}'.format(**locals()), expr=expr)
 
             def eval_method():
                 if (variable, member) == ('console', 'debug'):
@@ -737,6 +742,13 @@ class JSInterpreter(object):
                         return obj.index(idx, start)
                     except ValueError:
                         return -1
+                elif member == 'charCodeAt':
+                    assertion(isinstance(obj, compat_str), 'must be applied on a string')
+                    # assertion(len(argvals) == 1, 'takes exactly one argument') # but not enforced
+                    idx = argvals[0] if isinstance(argvals[0], int) else 0
+                    if idx >= len(obj):
+                        return None
+                    return ord(obj[idx])
 
                 idx = int(member) if isinstance(obj, list) else member
                 return obj[idx](argvals, allow_recursion=allow_recursion)
@@ -820,12 +832,10 @@ class JSInterpreter(object):
             if mobj is None:
                 break
             start, body_start = mobj.span()
-            body, remaining = self._separate_at_paren(code[body_start - 1:])
-            name = self._named_object(
-                local_vars,
-                self.extract_function_from_code(
-                    self.build_arglist(mobj.group('args')),
-                    body, local_vars, *global_stack))
+            body, remaining = self._separate_at_paren(code[body_start - 1:], '}')
+            name = self._named_object(local_vars, self.extract_function_from_code(
+                [x.strip() for x in mobj.group('args').split(',')],
+                body, local_vars, *global_stack))
             code = code[:start] + name + remaining
         return self.build_function(argnames, code, local_vars, *global_stack)
 
@@ -854,7 +864,7 @@ class JSInterpreter(object):
                 zip_longest(argnames, args, fillvalue=None))
             global_stack[0].update(kwargs)
             var_stack = LocalNameSpace(*global_stack)
-            ret, should_abort = self.interpret_statement(code.replace('\n', ''), var_stack, allow_recursion - 1)
+            ret, should_abort = self.interpret_statement(code.replace('\n', ' '), var_stack, allow_recursion - 1)
             if should_abort:
                 return ret
         return resf

From 218c423bc042674a8834ffc09520a94fbbe7b138 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 1 Sep 2022 13:28:30 +0100
Subject: [PATCH 537/860] [cache] Add cache validation by program version,
 based on yt-dlp

---
 test/test_cache.py  | 16 ++++++++++++++--
 youtube_dl/cache.py | 28 +++++++++++++++++++++++-----
 2 files changed, 37 insertions(+), 7 deletions(-)

diff --git a/test/test_cache.py b/test/test_cache.py
index a16160142..931074aa1 100644
--- a/test/test_cache.py
+++ b/test/test_cache.py
@@ -3,17 +3,18 @@
 
 from __future__ import unicode_literals
 
-import shutil
-
 # Allow direct execution
 import os
 import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+import shutil
 
 from test.helper import FakeYDL
 from youtube_dl.cache import Cache
+from youtube_dl.utils import version_tuple
+from youtube_dl.version import __version__
 
 
 def _is_empty(d):
@@ -54,6 +55,17 @@ class TestCache(unittest.TestCase):
         self.assertFalse(os.path.exists(self.test_dir))
         self.assertEqual(c.load('test_cache', 'k.'), None)
 
+    def test_cache_validation(self):
+        ydl = FakeYDL({
+            'cachedir': self.test_dir,
+        })
+        c = Cache(ydl)
+        obj = {'x': 1, 'y': ['ä', '\\a', True]}
+        c.store('test_cache', 'k.', obj)
+        self.assertEqual(c.load('test_cache', 'k.', min_ver='1970.01.01'), obj)
+        new_version = '.'.join(('%d' % ((v + 1) if i == 0 else v, )) for i, v in enumerate(version_tuple(__version__)))
+        self.assertIs(c.load('test_cache', 'k.', min_ver=new_version), None)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/cache.py b/youtube_dl/cache.py
index 7bdade1bd..4822439d0 100644
--- a/youtube_dl/cache.py
+++ b/youtube_dl/cache.py
@@ -10,12 +10,21 @@ import traceback
 
 from .compat import compat_getenv
 from .utils import (
+    error_to_compat_str,
     expand_path,
+    is_outdated_version,
+    try_get,
     write_json_file,
 )
+from .version import __version__
 
 
 class Cache(object):
+
+    _YTDL_DIR = 'youtube-dl'
+    _VERSION_KEY = _YTDL_DIR + '_version'
+    _DEFAULT_VERSION = '2021.12.17'
+
     def __init__(self, ydl):
         self._ydl = ydl
 
@@ -23,7 +32,7 @@ class Cache(object):
         res = self._ydl.params.get('cachedir')
         if res is None:
             cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
-            res = os.path.join(cache_root, 'youtube-dl')
+            res = os.path.join(cache_root, self._YTDL_DIR)
         return expand_path(res)
 
     def _get_cache_fn(self, section, key, dtype):
@@ -50,13 +59,22 @@ class Cache(object):
             except OSError as ose:
                 if ose.errno != errno.EEXIST:
                     raise
-            write_json_file(data, fn)
+            write_json_file({self._VERSION_KEY: __version__, 'data': data}, fn)
         except Exception:
             tb = traceback.format_exc()
             self._ydl.report_warning(
                 'Writing cache to %r failed: %s' % (fn, tb))
 
-    def load(self, section, key, dtype='json', default=None):
+    def _validate(self, data, min_ver):
+        version = try_get(data, lambda x: x[self._VERSION_KEY])
+        if not version:  # Backward compatibility
+            data, version = {'data': data}, self._DEFAULT_VERSION
+        if not is_outdated_version(version, min_ver or '0', assume_new=False):
+            return data['data']
+        self._ydl.to_screen(
+            'Discarding old cache from version {version} (needs {min_ver})'.format(**locals()))
+
+    def load(self, section, key, dtype='json', default=None, min_ver=None):
         assert dtype in ('json',)
 
         if not self.enabled:
@@ -66,12 +84,12 @@ class Cache(object):
         try:
             try:
                 with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
-                    return json.load(cachef)
+                    return self._validate(json.load(cachef), min_ver)
             except ValueError:
                 try:
                     file_size = os.path.getsize(cache_fn)
                 except (OSError, IOError) as oe:
-                    file_size = str(oe)
+                    file_size = error_to_compat_str(oe)
                 self._ydl.report_warning(
                     'Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
         except IOError:

From 7009bb9f3182449ae8cc05cc28b768b63030a485 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 2 Sep 2022 20:41:39 +0530
Subject: [PATCH 538/860] [jsinterp] Workaround operator associativity issue *
 temporary fix for player 5a3b6271 [1]

1. https://github.com/yt-dlp/yt-dlp/issues/4635#issuecomment-1235384480
---
 test/test_youtube_signature.py | 4 ++++
 youtube_dl/jsinterp.py         | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index ec914a871..4e678cae0 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -131,6 +131,10 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/c57c113c/player_ias.vflset/en_US/base.js',
         '-Txvy6bT5R6LqgnQNx', 'dcklJCnRUHbgSg',
     ),
+    (
+        'https://www.youtube.com/s/player/5a3b6271/player_ias.vflset/en_US/base.js',
+        'B2j7f_UPT4rfje85Lu_e', 'm5DmNymaGQ5RdQ',
+    ),
 ]
 
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index d13329396..99dd98435 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -107,8 +107,8 @@ _OPERATORS = (
     ('+', _js_arith_op(operator.add)),
     ('-', _js_arith_op(operator.sub)),
     ('*', _js_arith_op(operator.mul)),
-    ('/', _js_div),
     ('%', _js_mod),
+    ('/', _js_div),
     ('**', _js_exp),
 )
 

From 9493ffdb8b690732e995422621bad3ed6c9041f5 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 4 Oct 2022 00:42:15 +0100
Subject: [PATCH 539/860] [test] Use windows-2019 for tests (At least for now)
 resolves #31249

---
 .github/workflows/ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 90bd63c32..a609f3704 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -15,12 +15,12 @@ jobs:
         run-tests-ext: [sh]
         include:
         # python 3.2 is only available on windows via setup-python
-        - os: windows-latest
+        - os: windows-2019
           python-version: 3.2
           python-impl: cpython
           ytdl-test-set: core
           run-tests-ext: bat
-        - os: windows-latest
+        - os: windows-2019
           python-version: 3.2
           python-impl: cpython
           ytdl-test-set: download

From d35557a75d943865e40410d51bfcc18276e98532 Mon Sep 17 00:00:00 2001
From: coletdjnz <coletdjnz@protonmail.com>
Date: Fri, 23 Sep 2022 12:10:35 +1200
Subject: [PATCH 540/860] [Telegraaf] Use mobile GraphQL API endpoint

Workaround for Cloudflare 403
Fixes https://github.com/yt-dlp/yt-dlp/issues/5000
Authored by: coletdjnz
---
 youtube_dl/extractor/telegraaf.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/telegraaf.py b/youtube_dl/extractor/telegraaf.py
index 2dc020537..5174898f2 100644
--- a/youtube_dl/extractor/telegraaf.py
+++ b/youtube_dl/extractor/telegraaf.py
@@ -34,7 +34,9 @@ class TelegraafIE(InfoExtractor):
         article_id = self._match_id(url)
 
         video_id = self._download_json(
-            'https://www.telegraaf.nl/graphql', article_id, query={
+            'https://app.telegraaf.nl/graphql', article_id,
+            headers={'User-Agent': 'De Telegraaf/6.8.11 (Android 11; en_US)'},
+            query={
                 'query': '''{
   article(uid: %s) {
     videos {

From 22127b271c8f3e9266840bc5a2fb994d6248e369 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 10 Oct 2022 17:41:40 +0000
Subject: [PATCH 541/860] [NRK] Remove explicit Accept-Encoding header that
 invites Brotli

Fixes #31285
---
 youtube_dl/extractor/nrk.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/nrk.py b/youtube_dl/extractor/nrk.py
index 6d01a25c3..5a62b50fc 100644
--- a/youtube_dl/extractor/nrk.py
+++ b/youtube_dl/extractor/nrk.py
@@ -60,8 +60,7 @@ class NRKBaseIE(InfoExtractor):
         return self._download_json(
             urljoin('https://psapi.nrk.no/', path),
             video_id, note or 'Downloading %s JSON' % item,
-            fatal=fatal, query=query,
-            headers={'Accept-Encoding': 'gzip, deflate, br'})
+            fatal=fatal, query=query)
 
 
 class NRKIE(NRKBaseIE):

From 1b1442887e67b63545453e10816904e2b4c561c1 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 10 Oct 2022 19:26:32 +0100
Subject: [PATCH 542/860] [manyvids] Improve extraction (#31172)

* extract all formats from page
* extract description, uploader, views, likes
* downrate previews
* fix tests
* use txt_or_none()
---
 youtube_dl/extractor/manyvids.py | 113 +++++++++++++++++++++++++------
 1 file changed, 91 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/manyvids.py b/youtube_dl/extractor/manyvids.py
index e8d7163e4..6805102ba 100644
--- a/youtube_dl/extractor/manyvids.py
+++ b/youtube_dl/extractor/manyvids.py
@@ -1,11 +1,16 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import re
+
 from .common import InfoExtractor
+from ..compat import compat_str
 from ..utils import (
     determine_ext,
+    extract_attributes,
     int_or_none,
     str_to_int,
+    url_or_none,
     urlencode_postdata,
 )
 
@@ -20,17 +25,20 @@ class ManyVidsIE(InfoExtractor):
             'id': '133957',
             'ext': 'mp4',
             'title': 'everthing about me (Preview)',
+            'uploader': 'ellyxxix',
             'view_count': int,
             'like_count': int,
         },
     }, {
         # full video
         'url': 'https://www.manyvids.com/Video/935718/MY-FACE-REVEAL/',
-        'md5': 'f3e8f7086409e9b470e2643edb96bdcc',
+        'md5': 'bb47bab0e0802c2a60c24ef079dfe60f',
         'info_dict': {
             'id': '935718',
             'ext': 'mp4',
             'title': 'MY FACE REVEAL',
+            'description': 'md5:ec5901d41808b3746fed90face161612',
+            'uploader': 'Sarah Calanthe',
             'view_count': int,
             'like_count': int,
         },
@@ -41,15 +49,43 @@ class ManyVidsIE(InfoExtractor):
 
         webpage = self._download_webpage(url, video_id)
 
-        video_url = self._search_regex(
-            r'data-(?:video-filepath|meta-video)\s*=s*(["\'])(?P<url>(?:(?!\1).)+)\1',
-            webpage, 'video URL', group='url')
+        info = self._search_regex(
+            r'''(<div\b[^>]*\bid\s*=\s*(['"])pageMetaDetails\2[^>]*>)''',
+            webpage, 'meta details', default='')
+        info = extract_attributes(info)
 
-        title = self._html_search_regex(
-            (r'<span[^>]+class=["\']item-title[^>]+>([^<]+)',
-             r'<h2[^>]+class=["\']h2 m-0["\'][^>]*>([^<]+)'),
-            webpage, 'title', default=None) or self._html_search_meta(
-            'twitter:title', webpage, 'title', fatal=True)
+        player = self._search_regex(
+            r'''(<div\b[^>]*\bid\s*=\s*(['"])rmpPlayerStream\2[^>]*>)''',
+            webpage, 'player details', default='')
+        player = extract_attributes(player)
+
+        video_urls_and_ids = (
+            (info.get('data-meta-video'), 'video'),
+            (player.get('data-video-transcoded'), 'transcoded'),
+            (player.get('data-video-filepath'), 'filepath'),
+            (self._og_search_video_url(webpage, secure=False, default=None), 'og_video'),
+        )
+
+        def txt_or_none(s, default=None):
+            return (s.strip() or default) if isinstance(s, compat_str) else default
+
+        uploader = txt_or_none(info.get('data-meta-author'))
+
+        def mung_title(s):
+            if uploader:
+                s = re.sub(r'^\s*%s\s+[|-]' % (re.escape(uploader), ), '', s)
+            return txt_or_none(s)
+
+        title = (
+            mung_title(info.get('data-meta-title'))
+            or self._html_search_regex(
+                (r'<span[^>]+class=["\']item-title[^>]+>([^<]+)',
+                 r'<h2[^>]+class=["\']h2 m-0["\'][^>]*>([^<]+)'),
+                webpage, 'title', default=None)
+            or self._html_search_meta(
+                'twitter:title', webpage, 'title', fatal=True))
+
+        title = re.sub(r'\s*[|-]\s+ManyVids\s*$', '', title) or title
 
         if any(p in webpage for p in ('preview_videos', '_preview.mp4')):
             title += ' (Preview)'
@@ -70,23 +106,56 @@ class ManyVidsIE(InfoExtractor):
                     'X-Requested-With': 'XMLHttpRequest'
                 })
 
-        if determine_ext(video_url) == 'm3u8':
-            formats = self._extract_m3u8_formats(
-                video_url, video_id, 'mp4', entry_protocol='m3u8_native',
-                m3u8_id='hls')
-        else:
-            formats = [{'url': video_url}]
+        formats = []
+        for v_url, fmt in video_urls_and_ids:
+            v_url = url_or_none(v_url)
+            if not v_url:
+                continue
+            if determine_ext(v_url) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    v_url, video_id, 'mp4', entry_protocol='m3u8_native',
+                    m3u8_id='hls'))
+            else:
+                formats.append({
+                    'url': v_url,
+                    'format_id': fmt,
+                })
 
-        like_count = int_or_none(self._search_regex(
-            r'data-likes=["\'](\d+)', webpage, 'like count', default=None))
-        view_count = str_to_int(self._html_search_regex(
-            r'(?s)<span[^>]+class="views-wrapper"[^>]*>(.+?)</span', webpage,
-            'view count', default=None))
+        self._remove_duplicate_formats(formats)
+
+        for f in formats:
+            if f.get('height') is None:
+                f['height'] = int_or_none(
+                    self._search_regex(r'_(\d{2,3}[02468])_', f['url'], 'video height', default=None))
+            if '/preview/' in f['url']:
+                f['format_id'] = '_'.join(filter(None, (f.get('format_id'), 'preview')))
+                f['preference'] = -10
+            if 'transcoded' in f['format_id']:
+                f['preference'] = f.get('preference', -1) - 1
+
+        self._sort_formats(formats)
+
+        def get_likes():
+            likes = self._search_regex(
+                r'''(<a\b[^>]*\bdata-id\s*=\s*(['"])%s\2[^>]*>)''' % (video_id, ),
+                webpage, 'likes', default='')
+            likes = extract_attributes(likes)
+            return int_or_none(likes.get('data-likes'))
+
+        def get_views():
+            return str_to_int(self._html_search_regex(
+                r'''(?s)<span\b[^>]*\bclass\s*=["']views-wrapper\b[^>]+>.+?<span\b[^>]+>\s*(\d[\d,.]*)\s*</span>''',
+                webpage, 'view count', default=None))
 
         return {
             'id': video_id,
             'title': title,
-            'view_count': view_count,
-            'like_count': like_count,
             'formats': formats,
+            'description': txt_or_none(info.get('data-meta-description')),
+            'uploader': txt_or_none(info.get('data-meta-author')),
+            'thumbnail': (
+                url_or_none(info.get('data-meta-image'))
+                or url_or_none(player.get('data-video-screenshot'))),
+            'view_count': get_views(),
+            'like_count': get_likes(),
         }

From 82e4eca711a128138ed0b84ddb4321e403d56340 Mon Sep 17 00:00:00 2001
From: Xiyue <113869642+xiyue077@users.noreply.github.com>
Date: Tue, 11 Oct 2022 09:52:48 +1100
Subject: [PATCH 543/860] [motherless] Fixed the broken uploader_id in the
 extractor (#31243)

* Fixed the broken uploader_id in the extractor.
* Make uploader_id RE looser
* Fix uploader_id in test Motherless_3
* Fix group pagination
* # coding: utf-8

Co-authored-by: Andy Xuming <xuminic@gmail.com>
Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/motherless.py | 13 +++++++------
 1 file changed, 7 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index ef1e081f2..35d2b46ed 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import datetime
@@ -71,7 +72,7 @@ class MotherlessIE(InfoExtractor):
             'title': 'a/ Hot Teens',
             'categories': list,
             'upload_date': '20210104',
-            'uploader_id': 'yonbiw',
+            'uploader_id': 'anonymous',
             'thumbnail': r're:https?://.*\.jpg',
             'age_limit': 18,
         },
@@ -127,7 +128,7 @@ class MotherlessIE(InfoExtractor):
 
         comment_count = webpage.count('class="media-comment-contents"')
         uploader_id = self._html_search_regex(
-            r'"thumb-member-username">\s+<a href="/m/([^"]+)"',
+            r'''(?s)['"](?:media-meta-member|thumb-member-username)\b[^>]+>\s*<a\b[^>]+\bhref\s*=\s*['"]/m/([^"']+)''',
             webpage, 'uploader_id')
 
         categories = self._html_search_meta('keywords', webpage, default=None)
@@ -169,7 +170,7 @@ class MotherlessGroupIE(InfoExtractor):
             'description': 'Sex can be funny. Wide smiles,laugh, games, fun of '
                            'any kind!'
         },
-        'playlist_mincount': 9,
+        'playlist_mincount': 0,
     }]
 
     @classmethod
@@ -208,9 +209,9 @@ class MotherlessGroupIE(InfoExtractor):
             r'<title>([\w\s]+\w)\s+-', webpage, 'title', fatal=False)
         description = self._html_search_meta(
             'description', webpage, fatal=False)
-        page_count = self._int(self._search_regex(
-            r'(\d+)</(?:a|span)><(?:a|span)[^>]+>\s*NEXT',
-            webpage, 'page_count'), 'page_count')
+        page_count = str_to_int(self._search_regex(
+            r'(\d+)\s*</(?:a|span)>\s*<(?:a|span)[^>]+(?:>\s*NEXT|\brel\s*=\s*["\']?next)\b',
+            webpage, 'page_count', default='1'))
         PAGE_SIZE = 80
 
         def _get_page(idx):

From 2ced5a79128f53faad94dc494d05925eb957c414 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 9 Aug 2022 19:34:34 +0100
Subject: [PATCH 544/860] [test] Implement string "lambda x: condition(x)" as
 an expected value

Semantics equivalent to `assert condition(got)`
---
 test/helper.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index e62aab11e..c6a2f0667 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -128,6 +128,12 @@ def expect_value(self, got, expected, field):
         self.assertTrue(
             contains_str in got,
             'field %s (value: %r) should contain %r' % (field, got, contains_str))
+    elif isinstance(expected, compat_str) and re.match(r'^lambda \w+:', expected):
+        fn = eval(expected)
+        suite = expected.split(':', 1)[1].strip()
+        self.assertTrue(
+            fn(got),
+            'Expected field %s to meet condition %s, but value %r failed ' % (field, suite, got))
     elif isinstance(expected, type):
         self.assertTrue(
             isinstance(got, expected),
@@ -137,7 +143,7 @@ def expect_value(self, got, expected, field):
     elif isinstance(expected, list) and isinstance(got, list):
         self.assertEqual(
             len(expected), len(got),
-            'Expect a list of length %d, but got a list of length %d for field %s' % (
+            'Expected a list of length %d, but got a list of length %d for field %s' % (
                 len(expected), len(got), field))
         for index, (item_got, item_expected) in enumerate(zip(got, expected)):
             type_got = type(item_got)

From c282e5f8d723763ba88c521221e4535f46453949 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 9 Aug 2022 19:37:58 +0100
Subject: [PATCH 545/860] [ZDF] Overhaul ZDF extractors * pull some yt-dlp
 changes into ZDFBaseIE._extract_format() * add test cases from yt-dlp to
 ZDFIE * fix crash in ZDFIE._extract_mobile() when object had no `formitaeten`
 * improve title extraction in ZDFChannelIE (remove trailing station ident) *
 avoid extracting non-video playlist items (fixes #31149)

---
 youtube_dl/extractor/zdf.py | 169 +++++++++++++++++++++---------------
 1 file changed, 97 insertions(+), 72 deletions(-)

diff --git a/youtube_dl/extractor/zdf.py b/youtube_dl/extractor/zdf.py
index 3d39bb33a..fcc63ef52 100644
--- a/youtube_dl/extractor/zdf.py
+++ b/youtube_dl/extractor/zdf.py
@@ -8,13 +8,14 @@ from ..compat import compat_str
 from ..utils import (
     determine_ext,
     ExtractorError,
+    extract_attributes,
     float_or_none,
     int_or_none,
     merge_dicts,
     NO_DEFAULT,
-    orderedSet,
     parse_codecs,
     qualities,
+    str_or_none,
     try_get,
     unified_timestamp,
     update_url_query,
@@ -57,28 +58,39 @@ class ZDFBaseIE(InfoExtractor):
         format_urls.add(format_url)
         mime_type = meta.get('mimeType')
         ext = determine_ext(format_url)
+
+        join_nonempty = lambda s, l: s.join(filter(None, l))
+        meta_map = lambda t: map(lambda x: str_or_none(meta.get(x)), t)
+
         if mime_type == 'application/x-mpegURL' or ext == 'm3u8':
-            formats.extend(self._extract_m3u8_formats(
+            new_formats = self._extract_m3u8_formats(
                 format_url, video_id, 'mp4', m3u8_id='hls',
-                entry_protocol='m3u8_native', fatal=False))
+                entry_protocol='m3u8_native', fatal=False)
         elif mime_type == 'application/f4m+xml' or ext == 'f4m':
-            formats.extend(self._extract_f4m_formats(
-                update_url_query(format_url, {'hdcore': '3.7.0'}), video_id, f4m_id='hds', fatal=False))
+            new_formats = self._extract_f4m_formats(
+                update_url_query(format_url, {'hdcore': '3.7.0'}), video_id, f4m_id='hds', fatal=False)
         else:
             f = parse_codecs(meta.get('mimeCodec'))
+            if not f:
+                data = meta.get('type', '').split('_')
+                if try_get(data, lambda x: x[2]) == ext:
+                    f = dict(zip(('vcodec', 'acodec'), data[1]))
+
             format_id = ['http']
-            for p in (meta.get('type'), meta.get('quality')):
-                if p and isinstance(p, compat_str):
-                    format_id.append(p)
+            format_id.extend(join_nonempty('-', meta_map(('type', 'quality'))))
             f.update({
                 'url': format_url,
                 'format_id': '-'.join(format_id),
-                'format_note': meta.get('quality'),
-                'language': meta.get('language'),
-                'quality': qualities(self._QUALITIES)(meta.get('quality')),
-                'preference': -10,
+                'tbr': int_or_none(self._search_regex(r'_(\d+)k_', format_url, 'tbr', default=None))
             })
-            formats.append(f)
+            new_formats = [f]
+
+        formats.extend(merge_dicts(f, {
+            'format_note': join_nonempty(',', meta_map(('quality', 'class'))),
+            'language': meta.get('language'),
+            'language_preference': 10 if meta.get('class') == 'main' else -10 if meta.get('class') == 'ad' else -1,
+            'quality': qualities(self._QUALITIES)(meta.get('quality')),
+        }) for f in new_formats)
 
     def _extract_ptmd(self, ptmd_url, video_id, api_token, referrer):
         ptmd = self._call_api(
@@ -107,6 +119,7 @@ class ZDFBaseIE(InfoExtractor):
                                 'type': f.get('type'),
                                 'mimeType': f.get('mimeType'),
                                 'quality': quality.get('quality'),
+                                'class': track.get('class'),
                                 'language': track.get('language'),
                             })
         self._sort_formats(formats)
@@ -171,6 +184,20 @@ class ZDFIE(ZDFBaseIE):
             'duration': 2615,
             'timestamp': 1465021200,
             'upload_date': '20160604',
+            'thumbnail': 'https://www.zdf.de/assets/mauve-im-labor-100~768x432?cb=1464909117806',
+        },
+    }, {
+        'url': 'https://www.zdf.de/funk/druck-11790/funk-alles-ist-verzaubert-102.html',
+        'md5': '1b93bdec7d02fc0b703c5e7687461628',
+        'info_dict': {
+            'ext': 'mp4',
+            'id': 'video_funk_1770473',
+            'duration': 1278,
+            'description': 'Die Neue an der Schule verdreht Ismail den Kopf.',
+            'title': 'Alles ist verzaubert',
+            'timestamp': 1635520560,
+            'upload_date': '20211029',
+            'thumbnail': 'https://www.zdf.de/assets/teaser-funk-alles-ist-verzaubert-100~1920x1080?cb=1636466431799',
         },
     }, {
         # Same as https://www.phoenix.de/sendungen/dokumentationen/gesten-der-maechtigen-i-a-89468.html?ref=suche
@@ -204,6 +231,19 @@ class ZDFIE(ZDFBaseIE):
             'timestamp': 1641355200,
             'upload_date': '20220105',
         },
+        'skip': 'No longer available "Diese Seite wurde leider nicht gefunden"'
+    }, {
+        'url': 'https://www.zdf.de/serien/soko-stuttgart/das-geld-anderer-leute-100.html',
+        'info_dict': {
+            'id': '191205_1800_sendung_sok8',
+            'ext': 'mp4',
+            'title': 'Das Geld anderer Leute',
+            'description': 'md5:cb6f660850dc5eb7d1ab776ea094959d',
+            'duration': 2581.0,
+            'timestamp': 1654790700,
+            'upload_date': '20220609',
+            'thumbnail': 'https://epg-image.zdf.de/fotobase-webdelivery/images/e2d7e55a-09f0-424e-ac73-6cac4dd65f35?layout=2400x1350',
+        },
     }]
 
     def _extract_entry(self, url, player, content, video_id):
@@ -265,15 +305,16 @@ class ZDFIE(ZDFBaseIE):
             'https://zdf-cdn.live.cellular.de/mediathekV2/document/%s' % video_id,
             video_id)
 
-        document = video['document']
-
-        title = document['titel']
-        content_id = document['basename']
-
         formats = []
-        format_urls = set()
-        for f in document['formitaeten']:
-            self._extract_format(content_id, formats, format_urls, f)
+        formitaeten = try_get(video, lambda x: x['document']['formitaeten'], list)
+        document = formitaeten and video['document']
+        if formitaeten:
+            title = document['titel']
+            content_id = document['basename']
+
+            format_urls = set()
+            for f in formitaeten or []:
+                self._extract_format(content_id, formats, format_urls, f)
         self._sort_formats(formats)
 
         thumbnails = []
@@ -320,9 +361,9 @@ class ZDFChannelIE(ZDFBaseIE):
         'url': 'https://www.zdf.de/sport/das-aktuelle-sportstudio',
         'info_dict': {
             'id': 'das-aktuelle-sportstudio',
-            'title': 'das aktuelle sportstudio | ZDF',
+            'title': 'das aktuelle sportstudio',
         },
-        'playlist_mincount': 23,
+        'playlist_mincount': 18,
     }, {
         'url': 'https://www.zdf.de/dokumentation/planet-e',
         'info_dict': {
@@ -330,6 +371,14 @@ class ZDFChannelIE(ZDFBaseIE):
             'title': 'planet e.',
         },
         'playlist_mincount': 50,
+    }, {
+        'url': 'https://www.zdf.de/gesellschaft/aktenzeichen-xy-ungeloest',
+        'info_dict': {
+            'id': 'aktenzeichen-xy-ungeloest',
+            'title': 'Aktenzeichen XY... ungelöst',
+            'entries': "lambda x: not any('xy580-fall1-kindermoerder-gesucht-100' in e['url'] for e in x)",
+        },
+        'playlist_mincount': 2,
     }, {
         'url': 'https://www.zdf.de/filme/taunuskrimi/',
         'only_matching': True,
@@ -339,60 +388,36 @@ class ZDFChannelIE(ZDFBaseIE):
     def suitable(cls, url):
         return False if ZDFIE.suitable(url) else super(ZDFChannelIE, cls).suitable(url)
 
+    def _og_search_title(self, webpage, fatal=False):
+        title = super(ZDFChannelIE, self)._og_search_title(webpage, fatal=fatal)
+        return re.split(r'\s+[-|]\s+ZDF(?:mediathek)?$', title or '')[0] or None
+
     def _real_extract(self, url):
         channel_id = self._match_id(url)
 
         webpage = self._download_webpage(url, channel_id)
 
-        entries = [
-            self.url_result(item_url, ie=ZDFIE.ie_key())
-            for item_url in orderedSet(re.findall(
-                r'data-plusbar-url=["\'](http.+?\.html)', webpage))]
+        matches = re.finditer(
+            r'''<div\b[^>]*?\sdata-plusbar-id\s*=\s*(["'])(?P<p_id>[\w-]+)\1[^>]*?\sdata-plusbar-url=\1(?P<url>%s)\1''' % ZDFIE._VALID_URL,
+            webpage)
 
-        return self.playlist_result(
-            entries, channel_id, self._og_search_title(webpage, fatal=False))
+        if self._downloader.params.get('noplaylist', False):
+            entry = next(
+                (self.url_result(m.group('url'), ie=ZDFIE.ie_key()) for m in matches),
+                None)
+            self.to_screen('Downloading just the main video because of --no-playlist')
+            if entry:
+                return entry
+        else:
+            self.to_screen('Downloading playlist %s - add --no-playlist to download just the main video' % (channel_id, ))
 
-        r"""
-        player = self._extract_player(webpage, channel_id)
+        def check_video(m):
+            v_ref = self._search_regex(
+                r'''(<a\b[^>]*?\shref\s*=[^>]+?\sdata-target-id\s*=\s*(["'])%s\2[^>]*>)''' % (m.group('p_id'), ),
+                webpage, 'check id', default='')
+            v_ref = extract_attributes(v_ref)
+            return v_ref.get('data-target-video-type') != 'novideo'
 
-        channel_id = self._search_regex(
-            r'docId\s*:\s*(["\'])(?P<id>(?!\1).+?)\1', webpage,
-            'channel id', group='id')
-
-        channel = self._call_api(
-            'https://api.zdf.de/content/documents/%s.json' % channel_id,
-            player, url, channel_id)
-
-        items = []
-        for module in channel['module']:
-            for teaser in try_get(module, lambda x: x['teaser'], list) or []:
-                t = try_get(
-                    teaser, lambda x: x['http://zdf.de/rels/target'], dict)
-                if not t:
-                    continue
-                items.extend(try_get(
-                    t,
-                    lambda x: x['resultsWithVideo']['http://zdf.de/rels/search/results'],
-                    list) or [])
-            items.extend(try_get(
-                module,
-                lambda x: x['filterRef']['resultsWithVideo']['http://zdf.de/rels/search/results'],
-                list) or [])
-
-        entries = []
-        entry_urls = set()
-        for item in items:
-            t = try_get(item, lambda x: x['http://zdf.de/rels/target'], dict)
-            if not t:
-                continue
-            sharing_url = t.get('http://zdf.de/rels/sharing-url')
-            if not sharing_url or not isinstance(sharing_url, compat_str):
-                continue
-            if sharing_url in entry_urls:
-                continue
-            entry_urls.add(sharing_url)
-            entries.append(self.url_result(
-                sharing_url, ie=ZDFIE.ie_key(), video_id=t.get('id')))
-
-        return self.playlist_result(entries, channel_id, channel.get('title'))
-        """
+        return self.playlist_from_matches(
+            (m.group('url') for m in matches if check_video(m)),
+            channel_id, self._og_search_title(webpage, fatal=False))

From 6e2626f092c63a5fa22a31df409610b5deaf3968 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 11 Oct 2022 05:58:10 +0100
Subject: [PATCH 546/860] [JSInterp] Improve separation logic

Based on https://github.com/yt-dlp/yt-dlp/commit/0468a3b3253957bfbeb98b4a7c71542ff80e9e06
---
 youtube_dl/jsinterp.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 99dd98435..530a705b4 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -214,7 +214,7 @@ class JSInterpreter(object):
         def __init__(self, msg, *args, **kwargs):
             expr = kwargs.pop('expr', None)
             if expr is not None:
-                msg = '{0} in: {1!r}'.format(msg.rstrip(), expr[:100])
+                msg = '{0} in: {1!r:.100}'.format(msg.rstrip(), expr)
             super(JSInterpreter.Exception, self).__init__(msg, *args, **kwargs)
 
     @classmethod
@@ -268,7 +268,7 @@ class JSInterpreter(object):
                 elif in_quote == '/' and char in '[]':
                     in_regex_char_group = char == '['
             escaping = not escaping and in_quote and char == '\\'
-            after_op = not in_quote and (char in cls.OP_CHARS or (char.isspace() and after_op))
+            after_op = not in_quote and (char in cls.OP_CHARS or char == '[' or (char.isspace() and after_op))
 
             if char != delim[pos] or any(counters.values()) or in_quote:
                 pos = skipping = 0
@@ -301,7 +301,7 @@ class JSInterpreter(object):
         separated = list(cls._separate(expr, delim, 1))
 
         if len(separated) < 2:
-            raise cls.Exception('No terminating paren {delim} in {expr}'.format(**locals()))
+            raise cls.Exception('No terminating paren {delim} in {expr:.100}'.format(**locals()))
         return separated[0][1:].strip(), separated[1].strip()
 
     @staticmethod

From c94a459a248352fd97dccc79ed6604a558459bfd Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 11 Oct 2022 12:18:12 +0000
Subject: [PATCH 547/860] [utils] Sanitize look-alike Unicode glyphs in non-ID
 filename fields when --restrict-filenames

Implements https://github.com/ytdl-org/youtube-dl/issues/31216#issuecomment-1236102822, which has a test.
---
 youtube_dl/utils.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index fea38ed32..23a65a81c 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -33,6 +33,7 @@ import sys
 import tempfile
 import time
 import traceback
+import unicodedata
 import xml.etree.ElementTree
 import zlib
 
@@ -2118,6 +2119,9 @@ def sanitize_filename(s, restricted=False, is_id=False):
             return '_'
         return char
 
+    # Replace look-alike Unicode glyphs
+    if restricted and not is_id:
+        s = unicodedata.normalize('NFKC', s)
     # Handle timestamps
     s = re.sub(r'[0-9]+(?::[0-9]+)+', lambda m: m.group(0).replace(':', '_'), s)
     result = ''.join(map(replace_insane, s))

From 11b284c81fe2988813c817918536fc3a5630870a Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 11 Oct 2022 12:36:44 +0000
Subject: [PATCH 548/860] [Common:JWPlayer] Fix x1000 scaling error

See https://github.com/yt-dlp/yt-dlp/issues/5106#issuecomment-1264625161
---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 797c35fd5..1f33a1e06 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2844,7 +2844,7 @@ class InfoExtractor(object):
                     'url': source_url,
                     'width': int_or_none(source.get('width')),
                     'height': height,
-                    'tbr': int_or_none(source.get('bitrate')),
+                    'tbr': int_or_none(source.get('bitrate'), scale=1000),
                     'ext': ext,
                 }
                 if source_url.startswith('rtmp'):

From c91cbf60729af93c4677864aa6c8b74b576146ca Mon Sep 17 00:00:00 2001
From: Xie Yanbo <xieyanbo@gmail.com>
Date: Tue, 11 Oct 2022 20:55:09 +0800
Subject: [PATCH 549/860] [netease] Get netease music download url through
 player api (#31235)

* remove unplayable song from test
* compatible with python 2
* using standard User_Agent, fix imports
* use hash instead of long description
* fix lint
* fix hash
---
 test/test_aes.py                     |   9 +-
 youtube_dl/aes.py                    |  37 +++++++-
 youtube_dl/extractor/neteasemusic.py | 121 +++++++++++++++++++--------
 3 files changed, 128 insertions(+), 39 deletions(-)

diff --git a/test/test_aes.py b/test/test_aes.py
index cc89fb6ab..0f181466b 100644
--- a/test/test_aes.py
+++ b/test/test_aes.py
@@ -8,7 +8,7 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from youtube_dl.aes import aes_decrypt, aes_encrypt, aes_cbc_decrypt, aes_cbc_encrypt, aes_decrypt_text
+from youtube_dl.aes import aes_decrypt, aes_encrypt, aes_cbc_decrypt, aes_cbc_encrypt, aes_decrypt_text, aes_ecb_encrypt
 from youtube_dl.utils import bytes_to_intlist, intlist_to_bytes
 import base64
 
@@ -58,6 +58,13 @@ class TestAES(unittest.TestCase):
         decrypted = (aes_decrypt_text(encrypted, password, 32))
         self.assertEqual(decrypted, self.secret_msg)
 
+    def test_ecb_encrypt(self):
+        data = bytes_to_intlist(self.secret_msg)
+        encrypted = intlist_to_bytes(aes_ecb_encrypt(data, self.key))
+        self.assertEqual(
+            encrypted,
+            b'\xaa\x86]\x81\x97>\x02\x92\x9d\x1bR[[L/u\xd3&\xd1(h\xde{\x81\x94\xba\x02\xae\xbd\xa6\xd0:')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/aes.py b/youtube_dl/aes.py
index d0de2d93f..a94a41079 100644
--- a/youtube_dl/aes.py
+++ b/youtube_dl/aes.py
@@ -8,6 +8,18 @@ from .utils import bytes_to_intlist, intlist_to_bytes
 BLOCK_SIZE_BYTES = 16
 
 
+def pkcs7_padding(data):
+    """
+    PKCS#7 padding
+
+    @param {int[]} data        cleartext
+    @returns {int[]}           padding data
+    """
+
+    remaining_length = BLOCK_SIZE_BYTES - len(data) % BLOCK_SIZE_BYTES
+    return data + [remaining_length] * remaining_length
+
+
 def aes_ctr_decrypt(data, key, counter):
     """
     Decrypt with aes in counter mode
@@ -76,8 +88,7 @@ def aes_cbc_encrypt(data, key, iv):
     previous_cipher_block = iv
     for i in range(block_count):
         block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
-        remaining_length = BLOCK_SIZE_BYTES - len(block)
-        block += [remaining_length] * remaining_length
+        block = pkcs7_padding(block)
         mixed_block = xor(block, previous_cipher_block)
 
         encrypted_block = aes_encrypt(mixed_block, expanded_key)
@@ -88,6 +99,28 @@ def aes_cbc_encrypt(data, key, iv):
     return encrypted_data
 
 
+def aes_ecb_encrypt(data, key):
+    """
+    Encrypt with aes in ECB mode. Using PKCS#7 padding
+
+    @param {int[]} data        cleartext
+    @param {int[]} key         16/24/32-Byte cipher key
+    @returns {int[]}           encrypted data
+    """
+    expanded_key = key_expansion(key)
+    block_count = int(ceil(float(len(data)) / BLOCK_SIZE_BYTES))
+
+    encrypted_data = []
+    for i in range(block_count):
+        block = data[i * BLOCK_SIZE_BYTES: (i + 1) * BLOCK_SIZE_BYTES]
+        block = pkcs7_padding(block)
+
+        encrypted_block = aes_encrypt(block, expanded_key)
+        encrypted_data += encrypted_block
+
+    return encrypted_data
+
+
 def key_expansion(data):
     """
     Generate key schedule
diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index 978a05841..fad22a2cd 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -1,20 +1,31 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from hashlib import md5
 from base64 import b64encode
+from binascii import hexlify
 from datetime import datetime
+from hashlib import md5
+from random import randint
+import json
 import re
+import time
 
 from .common import InfoExtractor
+from ..aes import aes_ecb_encrypt, pkcs7_padding
 from ..compat import (
     compat_urllib_parse_urlencode,
     compat_str,
     compat_itertools_count,
 )
 from ..utils import (
-    sanitized_Request,
+    ExtractorError,
+    bytes_to_intlist,
     float_or_none,
+    int_or_none,
+    intlist_to_bytes,
+    sanitized_Request,
+    std_headers,
+    try_get,
 )
 
 
@@ -35,32 +46,85 @@ class NetEaseMusicBaseIE(InfoExtractor):
         result = b64encode(m.digest()).decode('ascii')
         return result.replace('/', '_').replace('+', '-')
 
+    @classmethod
+    def make_player_api_request_data_and_headers(cls, song_id, bitrate):
+        KEY = b'e82ckenh8dichen8'
+        URL = '/api/song/enhance/player/url'
+        now = int(time.time() * 1000)
+        rand = randint(0, 1000)
+        cookie = {
+            'osver': None,
+            'deviceId': None,
+            'appver': '8.0.0',
+            'versioncode': '140',
+            'mobilename': None,
+            'buildver': '1623435496',
+            'resolution': '1920x1080',
+            '__csrf': '',
+            'os': 'pc',
+            'channel': None,
+            'requestId': '{0}_{1:04}'.format(now, rand),
+        }
+        request_text = json.dumps(
+            {'ids': '[{0}]'.format(song_id), 'br': bitrate, 'header': cookie},
+            separators=(',', ':'))
+        message = 'nobody{0}use{1}md5forencrypt'.format(
+            URL, request_text).encode('latin1')
+        msg_digest = md5(message).hexdigest()
+
+        data = '{0}-36cd479b6b5-{1}-36cd479b6b5-{2}'.format(
+            URL, request_text, msg_digest)
+        data = pkcs7_padding(bytes_to_intlist(data))
+        encrypted = intlist_to_bytes(aes_ecb_encrypt(data, bytes_to_intlist(KEY)))
+        encrypted_params = hexlify(encrypted).decode('ascii').upper()
+
+        cookie = '; '.join(
+            ['{0}={1}'.format(k, v if v is not None else 'undefined')
+             for [k, v] in cookie.items()])
+
+        headers = {
+            'User-Agent': std_headers['User-Agent'],
+            'Content-Type': 'application/x-www-form-urlencoded',
+            'Referer': 'https://music.163.com',
+            'Cookie': cookie,
+        }
+        return ('params={0}'.format(encrypted_params), headers)
+
+    def _call_player_api(self, song_id, bitrate):
+        url = 'https://interface3.music.163.com/eapi/song/enhance/player/url'
+        data, headers = self.make_player_api_request_data_and_headers(song_id, bitrate)
+        try:
+            return self._download_json(
+                url, song_id, data=data.encode('ascii'), headers=headers)
+        except ExtractorError as e:
+            if type(e.cause) in (ValueError, TypeError):
+                # JSON load failure
+                raise
+        except Exception:
+            pass
+        return {}
+
     def extract_formats(self, info):
         formats = []
+        song_id = info['id']
         for song_format in self._FORMATS:
             details = info.get(song_format)
             if not details:
                 continue
-            song_file_path = '/%s/%s.%s' % (
-                self._encrypt(details['dfsId']), details['dfsId'], details['extension'])
 
-            # 203.130.59.9, 124.40.233.182, 115.231.74.139, etc is a reverse proxy-like feature
-            # from NetEase's CDN provider that can be used if m5.music.126.net does not
-            # work, especially for users outside of Mainland China
-            # via: https://github.com/JixunMoe/unblock-163/issues/3#issuecomment-163115880
-            for host in ('http://m5.music.126.net', 'http://115.231.74.139/m1.music.126.net',
-                         'http://124.40.233.182/m1.music.126.net', 'http://203.130.59.9/m1.music.126.net'):
-                song_url = host + song_file_path
+            bitrate = int_or_none(details.get('bitrate')) or 999000
+            data = self._call_player_api(song_id, bitrate)
+            for song in try_get(data, lambda x: x['data'], list) or []:
+                song_url = try_get(song, lambda x: x['url'])
                 if self._is_valid_url(song_url, info['id'], 'song'):
                     formats.append({
                         'url': song_url,
                         'ext': details.get('extension'),
-                        'abr': float_or_none(details.get('bitrate'), scale=1000),
+                        'abr': float_or_none(song.get('br'), scale=1000),
                         'format_id': song_format,
-                        'filesize': details.get('size'),
-                        'asr': details.get('sr')
+                        'filesize': int_or_none(song.get('size')),
+                        'asr': int_or_none(details.get('sr')),
                     })
-                    break
         return formats
 
     @classmethod
@@ -79,30 +143,16 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
     _VALID_URL = r'https?://music\.163\.com/(#/)?song\?id=(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://music.163.com/#/song?id=32102397',
-        'md5': 'f2e97280e6345c74ba9d5677dd5dcb45',
+        'md5': '3e909614ce09b1ccef4a3eb205441190',
         'info_dict': {
             'id': '32102397',
             'ext': 'mp3',
-            'title': 'Bad Blood (feat. Kendrick Lamar)',
+            'title': 'Bad Blood',
             'creator': 'Taylor Swift / Kendrick Lamar',
-            'upload_date': '20150517',
-            'timestamp': 1431878400,
-            'description': 'md5:a10a54589c2860300d02e1de821eb2ef',
+            'upload_date': '20150516',
+            'timestamp': 1431792000,
+            'description': 'md5:25fc5f27e47aad975aa6d36382c7833c',
         },
-        'skip': 'Blocked outside Mainland China',
-    }, {
-        'note': 'No lyrics translation.',
-        'url': 'http://music.163.com/#/song?id=29822014',
-        'info_dict': {
-            'id': '29822014',
-            'ext': 'mp3',
-            'title': '听见下雨的声音',
-            'creator': '周杰伦',
-            'upload_date': '20141225',
-            'timestamp': 1419523200,
-            'description': 'md5:a4d8d89f44656af206b7b2555c0bce6c',
-        },
-        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'No lyrics.',
         'url': 'http://music.163.com/song?id=17241424',
@@ -112,9 +162,9 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'title': 'Opus 28',
             'creator': 'Dustin O\'Halloran',
             'upload_date': '20080211',
+            'description': 'md5:f12945b0f6e0365e3b73c5032e1b0ff4',
             'timestamp': 1202745600,
         },
-        'skip': 'Blocked outside Mainland China',
     }, {
         'note': 'Has translated name.',
         'url': 'http://music.163.com/#/song?id=22735043',
@@ -128,7 +178,6 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'timestamp': 1264608000,
             'alt_title': '说出愿望吧(Genie)',
         },
-        'skip': 'Blocked outside Mainland China',
     }]
 
     def _process_lyrics(self, lyrics_info):

From 7bbd5b13d4c6cfc3e24f56413ff1a1eace8472b8 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 12 Oct 2022 01:09:55 +0100
Subject: [PATCH 550/860] [Motherless] Pull from yt-dlp, etc

* use username field
* loosen regexes
* warn on page count 0 in group
* avoid reloading group page 1
Closes #29626
---
 youtube_dl/extractor/motherless.py | 33 +++++++++++++++++++++++-------
 1 file changed, 26 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/motherless.py b/youtube_dl/extractor/motherless.py
index 35d2b46ed..d352cb180 100644
--- a/youtube_dl/extractor/motherless.py
+++ b/youtube_dl/extractor/motherless.py
@@ -126,9 +126,10 @@ class MotherlessIE(InfoExtractor):
                 kwargs = {_AGO_UNITS.get(uploaded_ago[-1]): delta}
                 upload_date = (datetime.datetime.utcnow() - datetime.timedelta(**kwargs)).strftime('%Y%m%d')
 
-        comment_count = webpage.count('class="media-comment-contents"')
+        comment_count = len(re.findall(r'''class\s*=\s*['"]media-comment-contents\b''', webpage))
         uploader_id = self._html_search_regex(
-            r'''(?s)['"](?:media-meta-member|thumb-member-username)\b[^>]+>\s*<a\b[^>]+\bhref\s*=\s*['"]/m/([^"']+)''',
+            (r'''<span\b[^>]+\bclass\s*=\s*["']username\b[^>]*>([^<]+)</span>''',
+             r'''(?s)['"](?:media-meta-member|thumb-member-username)\b[^>]+>\s*<a\b[^>]+\bhref\s*=\s*['"]/m/([^"']+)'''),
             webpage, 'uploader_id')
 
         categories = self._html_search_meta('keywords', webpage, default=None)
@@ -171,6 +172,17 @@ class MotherlessGroupIE(InfoExtractor):
                            'any kind!'
         },
         'playlist_mincount': 0,
+        'expected_warnings': [
+            'This group has no videos.',
+        ]
+    }, {
+        'url': 'https://motherless.com/g/beautiful_cock',
+        'info_dict': {
+            'id': 'beautiful_cock',
+            'title': 'Beautiful Cock',
+            'description': 'Group for lovely cocks yours, mine, a friends anything human',
+        },
+        'playlist_mincount': 2500,
     }]
 
     @classmethod
@@ -211,14 +223,21 @@ class MotherlessGroupIE(InfoExtractor):
             'description', webpage, fatal=False)
         page_count = str_to_int(self._search_regex(
             r'(\d+)\s*</(?:a|span)>\s*<(?:a|span)[^>]+(?:>\s*NEXT|\brel\s*=\s*["\']?next)\b',
-            webpage, 'page_count', default='1'))
+            webpage, 'page_count', default=0))
+        if not page_count:
+            message = self._search_regex(
+                r'''class\s*=\s*['"]error-page\b[^>]*>\s*<p[^>]*>\s*(?P<error_msg>[^<]+)(?<=\S)\s*''',
+                webpage, 'error_msg', default=None) or 'This group has no videos.'
+            self.report_warning(message, group_id)
+            page_count = 1
         PAGE_SIZE = 80
 
         def _get_page(idx):
-            webpage = self._download_webpage(
-                page_url, group_id, query={'page': idx + 1},
-                note='Downloading page %d/%d' % (idx + 1, page_count)
-            )
+            if idx > 0:
+                webpage = self._download_webpage(
+                    page_url, group_id, query={'page': idx + 1},
+                    note='Downloading page %d/%d' % (idx + 1, page_count)
+                )
             for entry in self._extract_entries(webpage, url):
                 yield entry
 

From 7135277fec497bd7649c31087aba52daa7897484 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 13 Oct 2022 01:59:01 +0000
Subject: [PATCH 551/860] [ManyVids] Support new single-page app structure

See https://github.com/yt-dlp/yt-dlp/issues/5210#issuecomment-1276919962.
---
 youtube_dl/extractor/manyvids.py | 10 ++++++++--
 1 file changed, 8 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/manyvids.py b/youtube_dl/extractor/manyvids.py
index 6805102ba..608a02a8d 100644
--- a/youtube_dl/extractor/manyvids.py
+++ b/youtube_dl/extractor/manyvids.py
@@ -47,7 +47,12 @@ class ManyVidsIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, video_id)
+        real_url = 'https://www.manyvids.com/video/%s/gtm.js' % (video_id, )
+        try:
+            webpage = self._download_webpage(real_url, video_id)
+        except:
+            # probably useless fallback
+            webpage = self._download_webpage(url, video_id)
 
         info = self._search_regex(
             r'''(<div\b[^>]*\bid\s*=\s*(['"])pageMetaDetails\2[^>]*>)''',
@@ -98,7 +103,8 @@ class ManyVidsIE(InfoExtractor):
             # Sets some cookies
             self._download_webpage(
                 'https://www.manyvids.com/includes/ajax_repository/you_had_me_at_hello.php',
-                video_id, fatal=False, data=urlencode_postdata({
+                video_id, note='Setting format cookies', fatal=False,
+                data=urlencode_postdata({
                     'mvtoken': mv_token,
                     'vid': video_id,
                 }), headers={

From ee8560d01eec511587f8207c3d84219ec620a9a6 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 13 Oct 2022 02:42:49 +0000
Subject: [PATCH 552/860] [ManyVids] Support new single-page app structure

---
 youtube_dl/extractor/manyvids.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/manyvids.py b/youtube_dl/extractor/manyvids.py
index 608a02a8d..75978cfd6 100644
--- a/youtube_dl/extractor/manyvids.py
+++ b/youtube_dl/extractor/manyvids.py
@@ -50,7 +50,7 @@ class ManyVidsIE(InfoExtractor):
         real_url = 'https://www.manyvids.com/video/%s/gtm.js' % (video_id, )
         try:
             webpage = self._download_webpage(real_url, video_id)
-        except:
+        except Exception:
             # probably useless fallback
             webpage = self._download_webpage(url, video_id)
 

From 447edc48e63f5f21797ea0d9ee84e37ed1547035 Mon Sep 17 00:00:00 2001
From: ache <ache@ache.one>
Date: Tue, 18 Oct 2022 15:06:27 +0000
Subject: [PATCH 553/860] Fix ADN extractor (#31275)

* Rename Anime Digital Network to Animation Digital Network, animationdigitalnetwork.fr
* Update the test to an available video
* Update the decoding key of subtitles
* Keep the support of old URLs
* Add a test to match the old URL
* Reduce redundancy of the URL name
* Fix md5 ^^"
* Fix undefined _BASE
* Process HTTP error text (eg geo-block) correctly and uniformly in Py3, Py2
* Skip test for CI since geo-blocked

Signed-off-by: ache <ache@ache.one>
Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/adn.py | 57 +++++++++++++++++++++----------------
 1 file changed, 32 insertions(+), 25 deletions(-)

diff --git a/youtube_dl/extractor/adn.py b/youtube_dl/extractor/adn.py
index a55ebbcbd..5ff419f19 100644
--- a/youtube_dl/extractor/adn.py
+++ b/youtube_dl/extractor/adn.py
@@ -31,30 +31,34 @@ from ..utils import (
 
 
 class ADNIE(InfoExtractor):
-    IE_DESC = 'Anime Digital Network'
-    _VALID_URL = r'https?://(?:www\.)?animedigitalnetwork\.fr/video/[^/]+/(?P<id>\d+)'
-    _TEST = {
-        'url': 'http://animedigitalnetwork.fr/video/blue-exorcist-kyoto-saga/7778-episode-1-debut-des-hostilites',
-        'md5': '0319c99885ff5547565cacb4f3f9348d',
+    IE_DESC = 'Animation Digital Network'
+    _VALID_URL = r'https?://(?:www\.)?(?:animation|anime)digitalnetwork\.fr/video/[^/]+/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://animationdigitalnetwork.fr/video/fruits-basket/9841-episode-1-a-ce-soir',
+        'md5': '1c9ef066ceb302c86f80c2b371615261',
         'info_dict': {
-            'id': '7778',
+            'id': '9841',
             'ext': 'mp4',
-            'title': 'Blue Exorcist - Kyôto Saga - Episode 1',
-            'description': 'md5:2f7b5aa76edbc1a7a92cedcda8a528d5',
-            'series': 'Blue Exorcist - Kyôto Saga',
-            'duration': 1467,
-            'release_date': '20170106',
+            'title': 'Fruits Basket - Episode 1',
+            'description': 'md5:14be2f72c3c96809b0ca424b0097d336',
+            'series': 'Fruits Basket',
+            'duration': 1437,
+            'release_date': '20190405',
             'comment_count': int,
             'average_rating': float,
-            'season_number': 2,
-            'episode': 'Début des hostilités',
+            'season_number': 1,
+            'episode': 'À ce soir !',
             'episode_number': 1,
-        }
-    }
+        },
+        'skip': 'Only available in region (FR, ...)',
+    }, {
+        'url': 'http://animedigitalnetwork.fr/video/blue-exorcist-kyoto-saga/7778-episode-1-debut-des-hostilites',
+        'only_matching': True,
+    }]
 
-    _NETRC_MACHINE = 'animedigitalnetwork'
-    _BASE_URL = 'http://animedigitalnetwork.fr'
-    _API_BASE_URL = 'https://gw.api.animedigitalnetwork.fr/'
+    _NETRC_MACHINE = 'animationdigitalnetwork'
+    _BASE = 'animationdigitalnetwork.fr'
+    _API_BASE_URL = 'https://gw.api.' + _BASE + '/'
     _PLAYER_BASE_URL = _API_BASE_URL + 'player/'
     _HEADERS = {}
     _LOGIN_ERR_MESSAGE = 'Unable to log in'
@@ -82,14 +86,14 @@ class ADNIE(InfoExtractor):
         if subtitle_location:
             enc_subtitles = self._download_webpage(
                 subtitle_location, video_id, 'Downloading subtitles data',
-                fatal=False, headers={'Origin': 'https://animedigitalnetwork.fr'})
+                fatal=False, headers={'Origin': 'https://' + self._BASE})
         if not enc_subtitles:
             return None
 
-        # http://animedigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js
+        # http://animationdigitalnetwork.fr/components/com_vodvideo/videojs/adn-vjs.min.js
         dec_subtitles = intlist_to_bytes(aes_cbc_decrypt(
             bytes_to_intlist(compat_b64decode(enc_subtitles[24:])),
-            bytes_to_intlist(binascii.unhexlify(self._K + 'ab9f52f5baae7c72')),
+            bytes_to_intlist(binascii.unhexlify(self._K + '7fac1178830cfe0c')),
             bytes_to_intlist(compat_b64decode(enc_subtitles[:24]))
         ))
         subtitles_json = self._parse_json(
@@ -138,9 +142,9 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
         if not username:
             return
         try:
+            url = self._API_BASE_URL + 'authentication/login'
             access_token = (self._download_json(
-                self._API_BASE_URL + 'authentication/login', None,
-                'Logging in', self._LOGIN_ERR_MESSAGE, fatal=False,
+                url, None, 'Logging in', self._LOGIN_ERR_MESSAGE, fatal=False,
                 data=urlencode_postdata({
                     'password': password,
                     'rememberMe': False,
@@ -153,7 +157,8 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
             message = None
             if isinstance(e.cause, compat_HTTPError) and e.cause.code == 401:
                 resp = self._parse_json(
-                    e.cause.read().decode(), None, fatal=False) or {}
+                    self._webpage_read_content(e.cause, url, username),
+                    username, fatal=False) or {}
                 message = resp.get('message') or resp.get('code')
             self.report_warning(message or self._LOGIN_ERR_MESSAGE)
 
@@ -211,7 +216,9 @@ Format: Marked,Start,End,Style,Name,MarginL,MarginR,MarginV,Effect,Text'''
                     # This usually goes away with a different random pkcs1pad, so retry
                     continue
 
-                error = self._parse_json(e.cause.read(), video_id)
+                error = self._parse_json(
+                    self._webpage_read_content(e.cause, links_url, video_id),
+                    video_id, fatal=False) or {}
                 message = error.get('message')
                 if e.cause.code == 403 and error.get('code') == 'player-bad-geolocation-country':
                     self.raise_geo_restricted(msg=message)

From 0faa45d6c08f518b73d20e341944ea7292f9f2b2 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 20 Oct 2022 11:06:44 +0000
Subject: [PATCH 554/860] [BongaCams] Support new .net domain

Resolves #31262.
---
 youtube_dl/extractor/bongacams.py | 18 +++++++++++++++++-
 1 file changed, 17 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/bongacams.py b/youtube_dl/extractor/bongacams.py
index 180542fbc..016999d55 100644
--- a/youtube_dl/extractor/bongacams.py
+++ b/youtube_dl/extractor/bongacams.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
@@ -12,13 +13,28 @@ from ..utils import (
 
 
 class BongaCamsIE(InfoExtractor):
-    _VALID_URL = r'https?://(?P<host>(?:[^/]+\.)?bongacams\d*\.com)/(?P<id>[^/?&#]+)'
+    _VALID_URL = r'https?://(?P<host>(?:[^/]+\.)?bongacams\d*\.(?:com|net))/(?P<id>[^/?&#]+)'
     _TESTS = [{
         'url': 'https://de.bongacams.com/azumi-8',
         'only_matching': True,
     }, {
         'url': 'https://cn.bongacams.com/azumi-8',
         'only_matching': True,
+    }, {
+        'url': 'https://de.bongacams.net/claireashton',
+        'info_dict': {
+            'id': 'claireashton',
+            'ext': 'mp4',
+            'title': r're:ClaireAshton \d{4}-\d{2}-\d{2} \d{2}:\d{2}',
+            'age_limit': 18,
+            'uploader_id': 'ClaireAshton',
+            'uploader': 'ClaireAshton',
+            'like_count': int,
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': True,
+        },
     }]
 
     def _real_extract(self, url):

From 502cefa41f1d24057b6158748b2072dc911af682 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 27 Oct 2022 14:33:00 +0000
Subject: [PATCH 555/860] [Vimeo] Update variable name in hydration JSON
 pattern

Fixes #31311
---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index a66912502..853b38402 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -663,7 +663,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
 
         if '//player.vimeo.com/video/' in url:
             config = self._parse_json(self._search_regex(
-                r'\bconfig\s*=\s*({.+?})\s*;', webpage, 'info section'), video_id)
+                r'\b(?:playerC|c)onfig\s*=\s*({.+?})\s*;', webpage, 'info section'), video_id)
             if config.get('view') == 4:
                 config = self._verify_player_video_password(
                     redirect_url, video_id, headers)

From d25cf62086443d86a633b8176b5c7e79f4cc569e Mon Sep 17 00:00:00 2001
From: Xie Yanbo <xieyanbo@gmail.com>
Date: Sun, 30 Oct 2022 19:46:46 +0800
Subject: [PATCH 556/860] [netease] Impove error handling (#31303)

* add warnings for users outside of China
* skip empty song urls

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/neteasemusic.py | 28 +++++++++++++++++++++++++---
 1 file changed, 25 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index fad22a2cd..2bbfc7858 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -20,6 +20,7 @@ from ..compat import (
 from ..utils import (
     ExtractorError,
     bytes_to_intlist,
+    error_to_compat_str,
     float_or_none,
     int_or_none,
     intlist_to_bytes,
@@ -94,17 +95,23 @@ class NetEaseMusicBaseIE(InfoExtractor):
         url = 'https://interface3.music.163.com/eapi/song/enhance/player/url'
         data, headers = self.make_player_api_request_data_and_headers(song_id, bitrate)
         try:
-            return self._download_json(
+            msg = 'empty result'
+            result = self._download_json(
                 url, song_id, data=data.encode('ascii'), headers=headers)
+            if result:
+                return result
         except ExtractorError as e:
             if type(e.cause) in (ValueError, TypeError):
                 # JSON load failure
                 raise
-        except Exception:
-            pass
+        except Exception as e:
+            msg = error_to_compat_str(e)
+            self.report_warning('%s API call (%s) failed: %s' % (
+                song_id, bitrate, msg))
         return {}
 
     def extract_formats(self, info):
+        err = 0
         formats = []
         song_id = info['id']
         for song_format in self._FORMATS:
@@ -116,6 +123,8 @@ class NetEaseMusicBaseIE(InfoExtractor):
             data = self._call_player_api(song_id, bitrate)
             for song in try_get(data, lambda x: x['data'], list) or []:
                 song_url = try_get(song, lambda x: x['url'])
+                if not song_url:
+                    continue
                 if self._is_valid_url(song_url, info['id'], 'song'):
                     formats.append({
                         'url': song_url,
@@ -125,6 +134,19 @@ class NetEaseMusicBaseIE(InfoExtractor):
                         'filesize': int_or_none(song.get('size')),
                         'asr': int_or_none(details.get('sr')),
                     })
+                elif err == 0:
+                    err = try_get(song, lambda x: x['code'], int)
+
+        if not formats:
+            msg = 'No media links found'
+            if err != 0 and (err < 200 or err >= 400):
+                raise ExtractorError(
+                    '%s (site code %d)' % (msg, err, ), expected=True)
+            else:
+                self.raise_geo_restricted(
+                    msg + ': probably this video is not available from your location due to geo restriction.',
+                    countries=['CN'])
+
         return formats
 
     @classmethod

From ce5d36486ea95b8961c639d118bad262c8d7a067 Mon Sep 17 00:00:00 2001
From: Xie Yanbo <xieyanbo@gmail.com>
Date: Sun, 30 Oct 2022 19:48:44 +0800
Subject: [PATCH 557/860] [netease] Support urls shared from mobile app
 (#31304)

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/neteasemusic.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/neteasemusic.py b/youtube_dl/extractor/neteasemusic.py
index 2bbfc7858..5e5c6271b 100644
--- a/youtube_dl/extractor/neteasemusic.py
+++ b/youtube_dl/extractor/neteasemusic.py
@@ -162,7 +162,7 @@ class NetEaseMusicBaseIE(InfoExtractor):
 class NetEaseMusicIE(NetEaseMusicBaseIE):
     IE_NAME = 'netease:song'
     IE_DESC = '网易云音乐'
-    _VALID_URL = r'https?://music\.163\.com/(#/)?song\?id=(?P<id>[0-9]+)'
+    _VALID_URL = r'https?://(y\.)?music\.163\.com/(?:[#m]/)?song\?.*?\bid=(?P<id>[0-9]+)'
     _TESTS = [{
         'url': 'http://music.163.com/#/song?id=32102397',
         'md5': '3e909614ce09b1ccef4a3eb205441190',
@@ -200,6 +200,18 @@ class NetEaseMusicIE(NetEaseMusicBaseIE):
             'timestamp': 1264608000,
             'alt_title': '说出愿望吧(Genie)',
         },
+    }, {
+        'url': 'https://y.music.163.com/m/song?app_version=8.8.45&id=95670&uct2=sKnvS4+0YStsWkqsPhFijw%3D%3D&dlt=0846',
+        'md5': '95826c73ea50b1c288b22180ec9e754d',
+        'info_dict': {
+            'id': '95670',
+            'ext': 'mp3',
+            'title': '国际歌',
+            'creator': '马备',
+            'upload_date': '19911130',
+            'timestamp': 691516800,
+            'description': 'md5:1ba2f911a2b0aa398479f595224f2141',
+        },
     }]
 
     def _process_lyrics(self, lyrics_info):

From a19855f0f50fe7a6eb05a1d8fee554897e4dbdda Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 31 Oct 2022 21:18:36 +0000
Subject: [PATCH 558/860] [compat] Add Python 2 Unicode casefold using a
 trivial wrapper around icu/CaseFolding.txt

---
 youtube_dl/casefold.py | 1643 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 1643 insertions(+)
 create mode 100644 youtube_dl/casefold.py

diff --git a/youtube_dl/casefold.py b/youtube_dl/casefold.py
new file mode 100644
index 000000000..546269a3c
--- /dev/null
+++ b/youtube_dl/casefold.py
@@ -0,0 +1,1643 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .compat import compat_str
+
+# CaseFolding-15.0.0.txt
+# Date: 2022-02-02, 23:35:35 GMT
+# © 2022 Unicode®, Inc.
+# Unicode and the Unicode Logo are registered trademarks of Unicode, Inc. in the U.S. and other countries.
+# For terms of use, see https://www.unicode.org/terms_of_use.html
+#
+# Unicode Character Database
+#   For documentation, see https://www.unicode.org/reports/tr44/
+#
+# Case Folding Properties
+#
+# This file is a supplement to the UnicodeData file.
+# It provides a case folding mapping generated from the Unicode Character Database.
+# If all characters are mapped according to the full mapping below, then
+# case differences (according to UnicodeData.txt and SpecialCasing.txt)
+# are eliminated.
+#
+# The data supports both implementations that require simple case foldings
+# (where string lengths don't change), and implementations that allow full case folding
+# (where string lengths may grow). Note that where they can be supported, the
+# full case foldings are superior: for example, they allow "MASSE" and "Maße" to match.
+#
+# All code points not listed in this file map to themselves.
+#
+# NOTE: case folding does not preserve normalization formats!
+#
+# For information on case folding, including how to have case folding
+# preserve normalization formats, see Section 3.13 Default Case Algorithms in
+# The Unicode Standard.
+#
+# ================================================================================
+# Format
+# ================================================================================
+# The entries in this file are in the following machine-readable format:
+#
+# <code>; <status>; <mapping>; # <name>
+#
+# The status field is:
+# C: common case folding, common mappings shared by both simple and full mappings.
+# F: full case folding, mappings that cause strings to grow in length. Multiple characters are separated by spaces.
+# S: simple case folding, mappings to single characters where different from F.
+# T: special case for uppercase I and dotted uppercase I
+#    - For non-Turkic languages, this mapping is normally not used.
+#    - For Turkic languages (tr, az), this mapping can be used instead of the normal mapping for these characters.
+#      Note that the Turkic mappings do not maintain canonical equivalence without additional processing.
+#      See the discussions of case mapping in the Unicode Standard for more information.
+#
+# Usage:
+#  A. To do a simple case folding, use the mappings with status C + S.
+#  B. To do a full case folding, use the mappings with status C + F.
+#
+#    The mappings with status T can be used or omitted depending on the desired case-folding
+#    behavior. (The default option is to exclude them.)
+#
+# =================================================================
+
+# Property: Case_Folding
+
+#  All code points not explicitly listed for Case_Folding
+#  have the value C for the status field, and the code point itself for the mapping field.
+
+# =================================================================
+_map_str = '''
+0041; C; 0061; # LATIN CAPITAL LETTER A
+0042; C; 0062; # LATIN CAPITAL LETTER B
+0043; C; 0063; # LATIN CAPITAL LETTER C
+0044; C; 0064; # LATIN CAPITAL LETTER D
+0045; C; 0065; # LATIN CAPITAL LETTER E
+0046; C; 0066; # LATIN CAPITAL LETTER F
+0047; C; 0067; # LATIN CAPITAL LETTER G
+0048; C; 0068; # LATIN CAPITAL LETTER H
+0049; C; 0069; # LATIN CAPITAL LETTER I
+0049; T; 0131; # LATIN CAPITAL LETTER I
+004A; C; 006A; # LATIN CAPITAL LETTER J
+004B; C; 006B; # LATIN CAPITAL LETTER K
+004C; C; 006C; # LATIN CAPITAL LETTER L
+004D; C; 006D; # LATIN CAPITAL LETTER M
+004E; C; 006E; # LATIN CAPITAL LETTER N
+004F; C; 006F; # LATIN CAPITAL LETTER O
+0050; C; 0070; # LATIN CAPITAL LETTER P
+0051; C; 0071; # LATIN CAPITAL LETTER Q
+0052; C; 0072; # LATIN CAPITAL LETTER R
+0053; C; 0073; # LATIN CAPITAL LETTER S
+0054; C; 0074; # LATIN CAPITAL LETTER T
+0055; C; 0075; # LATIN CAPITAL LETTER U
+0056; C; 0076; # LATIN CAPITAL LETTER V
+0057; C; 0077; # LATIN CAPITAL LETTER W
+0058; C; 0078; # LATIN CAPITAL LETTER X
+0059; C; 0079; # LATIN CAPITAL LETTER Y
+005A; C; 007A; # LATIN CAPITAL LETTER Z
+00B5; C; 03BC; # MICRO SIGN
+00C0; C; 00E0; # LATIN CAPITAL LETTER A WITH GRAVE
+00C1; C; 00E1; # LATIN CAPITAL LETTER A WITH ACUTE
+00C2; C; 00E2; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX
+00C3; C; 00E3; # LATIN CAPITAL LETTER A WITH TILDE
+00C4; C; 00E4; # LATIN CAPITAL LETTER A WITH DIAERESIS
+00C5; C; 00E5; # LATIN CAPITAL LETTER A WITH RING ABOVE
+00C6; C; 00E6; # LATIN CAPITAL LETTER AE
+00C7; C; 00E7; # LATIN CAPITAL LETTER C WITH CEDILLA
+00C8; C; 00E8; # LATIN CAPITAL LETTER E WITH GRAVE
+00C9; C; 00E9; # LATIN CAPITAL LETTER E WITH ACUTE
+00CA; C; 00EA; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX
+00CB; C; 00EB; # LATIN CAPITAL LETTER E WITH DIAERESIS
+00CC; C; 00EC; # LATIN CAPITAL LETTER I WITH GRAVE
+00CD; C; 00ED; # LATIN CAPITAL LETTER I WITH ACUTE
+00CE; C; 00EE; # LATIN CAPITAL LETTER I WITH CIRCUMFLEX
+00CF; C; 00EF; # LATIN CAPITAL LETTER I WITH DIAERESIS
+00D0; C; 00F0; # LATIN CAPITAL LETTER ETH
+00D1; C; 00F1; # LATIN CAPITAL LETTER N WITH TILDE
+00D2; C; 00F2; # LATIN CAPITAL LETTER O WITH GRAVE
+00D3; C; 00F3; # LATIN CAPITAL LETTER O WITH ACUTE
+00D4; C; 00F4; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX
+00D5; C; 00F5; # LATIN CAPITAL LETTER O WITH TILDE
+00D6; C; 00F6; # LATIN CAPITAL LETTER O WITH DIAERESIS
+00D8; C; 00F8; # LATIN CAPITAL LETTER O WITH STROKE
+00D9; C; 00F9; # LATIN CAPITAL LETTER U WITH GRAVE
+00DA; C; 00FA; # LATIN CAPITAL LETTER U WITH ACUTE
+00DB; C; 00FB; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX
+00DC; C; 00FC; # LATIN CAPITAL LETTER U WITH DIAERESIS
+00DD; C; 00FD; # LATIN CAPITAL LETTER Y WITH ACUTE
+00DE; C; 00FE; # LATIN CAPITAL LETTER THORN
+00DF; F; 0073 0073; # LATIN SMALL LETTER SHARP S
+0100; C; 0101; # LATIN CAPITAL LETTER A WITH MACRON
+0102; C; 0103; # LATIN CAPITAL LETTER A WITH BREVE
+0104; C; 0105; # LATIN CAPITAL LETTER A WITH OGONEK
+0106; C; 0107; # LATIN CAPITAL LETTER C WITH ACUTE
+0108; C; 0109; # LATIN CAPITAL LETTER C WITH CIRCUMFLEX
+010A; C; 010B; # LATIN CAPITAL LETTER C WITH DOT ABOVE
+010C; C; 010D; # LATIN CAPITAL LETTER C WITH CARON
+010E; C; 010F; # LATIN CAPITAL LETTER D WITH CARON
+0110; C; 0111; # LATIN CAPITAL LETTER D WITH STROKE
+0112; C; 0113; # LATIN CAPITAL LETTER E WITH MACRON
+0114; C; 0115; # LATIN CAPITAL LETTER E WITH BREVE
+0116; C; 0117; # LATIN CAPITAL LETTER E WITH DOT ABOVE
+0118; C; 0119; # LATIN CAPITAL LETTER E WITH OGONEK
+011A; C; 011B; # LATIN CAPITAL LETTER E WITH CARON
+011C; C; 011D; # LATIN CAPITAL LETTER G WITH CIRCUMFLEX
+011E; C; 011F; # LATIN CAPITAL LETTER G WITH BREVE
+0120; C; 0121; # LATIN CAPITAL LETTER G WITH DOT ABOVE
+0122; C; 0123; # LATIN CAPITAL LETTER G WITH CEDILLA
+0124; C; 0125; # LATIN CAPITAL LETTER H WITH CIRCUMFLEX
+0126; C; 0127; # LATIN CAPITAL LETTER H WITH STROKE
+0128; C; 0129; # LATIN CAPITAL LETTER I WITH TILDE
+012A; C; 012B; # LATIN CAPITAL LETTER I WITH MACRON
+012C; C; 012D; # LATIN CAPITAL LETTER I WITH BREVE
+012E; C; 012F; # LATIN CAPITAL LETTER I WITH OGONEK
+0130; F; 0069 0307; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0130; T; 0069; # LATIN CAPITAL LETTER I WITH DOT ABOVE
+0132; C; 0133; # LATIN CAPITAL LIGATURE IJ
+0134; C; 0135; # LATIN CAPITAL LETTER J WITH CIRCUMFLEX
+0136; C; 0137; # LATIN CAPITAL LETTER K WITH CEDILLA
+0139; C; 013A; # LATIN CAPITAL LETTER L WITH ACUTE
+013B; C; 013C; # LATIN CAPITAL LETTER L WITH CEDILLA
+013D; C; 013E; # LATIN CAPITAL LETTER L WITH CARON
+013F; C; 0140; # LATIN CAPITAL LETTER L WITH MIDDLE DOT
+0141; C; 0142; # LATIN CAPITAL LETTER L WITH STROKE
+0143; C; 0144; # LATIN CAPITAL LETTER N WITH ACUTE
+0145; C; 0146; # LATIN CAPITAL LETTER N WITH CEDILLA
+0147; C; 0148; # LATIN CAPITAL LETTER N WITH CARON
+0149; F; 02BC 006E; # LATIN SMALL LETTER N PRECEDED BY APOSTROPHE
+014A; C; 014B; # LATIN CAPITAL LETTER ENG
+014C; C; 014D; # LATIN CAPITAL LETTER O WITH MACRON
+014E; C; 014F; # LATIN CAPITAL LETTER O WITH BREVE
+0150; C; 0151; # LATIN CAPITAL LETTER O WITH DOUBLE ACUTE
+0152; C; 0153; # LATIN CAPITAL LIGATURE OE
+0154; C; 0155; # LATIN CAPITAL LETTER R WITH ACUTE
+0156; C; 0157; # LATIN CAPITAL LETTER R WITH CEDILLA
+0158; C; 0159; # LATIN CAPITAL LETTER R WITH CARON
+015A; C; 015B; # LATIN CAPITAL LETTER S WITH ACUTE
+015C; C; 015D; # LATIN CAPITAL LETTER S WITH CIRCUMFLEX
+015E; C; 015F; # LATIN CAPITAL LETTER S WITH CEDILLA
+0160; C; 0161; # LATIN CAPITAL LETTER S WITH CARON
+0162; C; 0163; # LATIN CAPITAL LETTER T WITH CEDILLA
+0164; C; 0165; # LATIN CAPITAL LETTER T WITH CARON
+0166; C; 0167; # LATIN CAPITAL LETTER T WITH STROKE
+0168; C; 0169; # LATIN CAPITAL LETTER U WITH TILDE
+016A; C; 016B; # LATIN CAPITAL LETTER U WITH MACRON
+016C; C; 016D; # LATIN CAPITAL LETTER U WITH BREVE
+016E; C; 016F; # LATIN CAPITAL LETTER U WITH RING ABOVE
+0170; C; 0171; # LATIN CAPITAL LETTER U WITH DOUBLE ACUTE
+0172; C; 0173; # LATIN CAPITAL LETTER U WITH OGONEK
+0174; C; 0175; # LATIN CAPITAL LETTER W WITH CIRCUMFLEX
+0176; C; 0177; # LATIN CAPITAL LETTER Y WITH CIRCUMFLEX
+0178; C; 00FF; # LATIN CAPITAL LETTER Y WITH DIAERESIS
+0179; C; 017A; # LATIN CAPITAL LETTER Z WITH ACUTE
+017B; C; 017C; # LATIN CAPITAL LETTER Z WITH DOT ABOVE
+017D; C; 017E; # LATIN CAPITAL LETTER Z WITH CARON
+017F; C; 0073; # LATIN SMALL LETTER LONG S
+0181; C; 0253; # LATIN CAPITAL LETTER B WITH HOOK
+0182; C; 0183; # LATIN CAPITAL LETTER B WITH TOPBAR
+0184; C; 0185; # LATIN CAPITAL LETTER TONE SIX
+0186; C; 0254; # LATIN CAPITAL LETTER OPEN O
+0187; C; 0188; # LATIN CAPITAL LETTER C WITH HOOK
+0189; C; 0256; # LATIN CAPITAL LETTER AFRICAN D
+018A; C; 0257; # LATIN CAPITAL LETTER D WITH HOOK
+018B; C; 018C; # LATIN CAPITAL LETTER D WITH TOPBAR
+018E; C; 01DD; # LATIN CAPITAL LETTER REVERSED E
+018F; C; 0259; # LATIN CAPITAL LETTER SCHWA
+0190; C; 025B; # LATIN CAPITAL LETTER OPEN E
+0191; C; 0192; # LATIN CAPITAL LETTER F WITH HOOK
+0193; C; 0260; # LATIN CAPITAL LETTER G WITH HOOK
+0194; C; 0263; # LATIN CAPITAL LETTER GAMMA
+0196; C; 0269; # LATIN CAPITAL LETTER IOTA
+0197; C; 0268; # LATIN CAPITAL LETTER I WITH STROKE
+0198; C; 0199; # LATIN CAPITAL LETTER K WITH HOOK
+019C; C; 026F; # LATIN CAPITAL LETTER TURNED M
+019D; C; 0272; # LATIN CAPITAL LETTER N WITH LEFT HOOK
+019F; C; 0275; # LATIN CAPITAL LETTER O WITH MIDDLE TILDE
+01A0; C; 01A1; # LATIN CAPITAL LETTER O WITH HORN
+01A2; C; 01A3; # LATIN CAPITAL LETTER OI
+01A4; C; 01A5; # LATIN CAPITAL LETTER P WITH HOOK
+01A6; C; 0280; # LATIN LETTER YR
+01A7; C; 01A8; # LATIN CAPITAL LETTER TONE TWO
+01A9; C; 0283; # LATIN CAPITAL LETTER ESH
+01AC; C; 01AD; # LATIN CAPITAL LETTER T WITH HOOK
+01AE; C; 0288; # LATIN CAPITAL LETTER T WITH RETROFLEX HOOK
+01AF; C; 01B0; # LATIN CAPITAL LETTER U WITH HORN
+01B1; C; 028A; # LATIN CAPITAL LETTER UPSILON
+01B2; C; 028B; # LATIN CAPITAL LETTER V WITH HOOK
+01B3; C; 01B4; # LATIN CAPITAL LETTER Y WITH HOOK
+01B5; C; 01B6; # LATIN CAPITAL LETTER Z WITH STROKE
+01B7; C; 0292; # LATIN CAPITAL LETTER EZH
+01B8; C; 01B9; # LATIN CAPITAL LETTER EZH REVERSED
+01BC; C; 01BD; # LATIN CAPITAL LETTER TONE FIVE
+01C4; C; 01C6; # LATIN CAPITAL LETTER DZ WITH CARON
+01C5; C; 01C6; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z WITH CARON
+01C7; C; 01C9; # LATIN CAPITAL LETTER LJ
+01C8; C; 01C9; # LATIN CAPITAL LETTER L WITH SMALL LETTER J
+01CA; C; 01CC; # LATIN CAPITAL LETTER NJ
+01CB; C; 01CC; # LATIN CAPITAL LETTER N WITH SMALL LETTER J
+01CD; C; 01CE; # LATIN CAPITAL LETTER A WITH CARON
+01CF; C; 01D0; # LATIN CAPITAL LETTER I WITH CARON
+01D1; C; 01D2; # LATIN CAPITAL LETTER O WITH CARON
+01D3; C; 01D4; # LATIN CAPITAL LETTER U WITH CARON
+01D5; C; 01D6; # LATIN CAPITAL LETTER U WITH DIAERESIS AND MACRON
+01D7; C; 01D8; # LATIN CAPITAL LETTER U WITH DIAERESIS AND ACUTE
+01D9; C; 01DA; # LATIN CAPITAL LETTER U WITH DIAERESIS AND CARON
+01DB; C; 01DC; # LATIN CAPITAL LETTER U WITH DIAERESIS AND GRAVE
+01DE; C; 01DF; # LATIN CAPITAL LETTER A WITH DIAERESIS AND MACRON
+01E0; C; 01E1; # LATIN CAPITAL LETTER A WITH DOT ABOVE AND MACRON
+01E2; C; 01E3; # LATIN CAPITAL LETTER AE WITH MACRON
+01E4; C; 01E5; # LATIN CAPITAL LETTER G WITH STROKE
+01E6; C; 01E7; # LATIN CAPITAL LETTER G WITH CARON
+01E8; C; 01E9; # LATIN CAPITAL LETTER K WITH CARON
+01EA; C; 01EB; # LATIN CAPITAL LETTER O WITH OGONEK
+01EC; C; 01ED; # LATIN CAPITAL LETTER O WITH OGONEK AND MACRON
+01EE; C; 01EF; # LATIN CAPITAL LETTER EZH WITH CARON
+01F0; F; 006A 030C; # LATIN SMALL LETTER J WITH CARON
+01F1; C; 01F3; # LATIN CAPITAL LETTER DZ
+01F2; C; 01F3; # LATIN CAPITAL LETTER D WITH SMALL LETTER Z
+01F4; C; 01F5; # LATIN CAPITAL LETTER G WITH ACUTE
+01F6; C; 0195; # LATIN CAPITAL LETTER HWAIR
+01F7; C; 01BF; # LATIN CAPITAL LETTER WYNN
+01F8; C; 01F9; # LATIN CAPITAL LETTER N WITH GRAVE
+01FA; C; 01FB; # LATIN CAPITAL LETTER A WITH RING ABOVE AND ACUTE
+01FC; C; 01FD; # LATIN CAPITAL LETTER AE WITH ACUTE
+01FE; C; 01FF; # LATIN CAPITAL LETTER O WITH STROKE AND ACUTE
+0200; C; 0201; # LATIN CAPITAL LETTER A WITH DOUBLE GRAVE
+0202; C; 0203; # LATIN CAPITAL LETTER A WITH INVERTED BREVE
+0204; C; 0205; # LATIN CAPITAL LETTER E WITH DOUBLE GRAVE
+0206; C; 0207; # LATIN CAPITAL LETTER E WITH INVERTED BREVE
+0208; C; 0209; # LATIN CAPITAL LETTER I WITH DOUBLE GRAVE
+020A; C; 020B; # LATIN CAPITAL LETTER I WITH INVERTED BREVE
+020C; C; 020D; # LATIN CAPITAL LETTER O WITH DOUBLE GRAVE
+020E; C; 020F; # LATIN CAPITAL LETTER O WITH INVERTED BREVE
+0210; C; 0211; # LATIN CAPITAL LETTER R WITH DOUBLE GRAVE
+0212; C; 0213; # LATIN CAPITAL LETTER R WITH INVERTED BREVE
+0214; C; 0215; # LATIN CAPITAL LETTER U WITH DOUBLE GRAVE
+0216; C; 0217; # LATIN CAPITAL LETTER U WITH INVERTED BREVE
+0218; C; 0219; # LATIN CAPITAL LETTER S WITH COMMA BELOW
+021A; C; 021B; # LATIN CAPITAL LETTER T WITH COMMA BELOW
+021C; C; 021D; # LATIN CAPITAL LETTER YOGH
+021E; C; 021F; # LATIN CAPITAL LETTER H WITH CARON
+0220; C; 019E; # LATIN CAPITAL LETTER N WITH LONG RIGHT LEG
+0222; C; 0223; # LATIN CAPITAL LETTER OU
+0224; C; 0225; # LATIN CAPITAL LETTER Z WITH HOOK
+0226; C; 0227; # LATIN CAPITAL LETTER A WITH DOT ABOVE
+0228; C; 0229; # LATIN CAPITAL LETTER E WITH CEDILLA
+022A; C; 022B; # LATIN CAPITAL LETTER O WITH DIAERESIS AND MACRON
+022C; C; 022D; # LATIN CAPITAL LETTER O WITH TILDE AND MACRON
+022E; C; 022F; # LATIN CAPITAL LETTER O WITH DOT ABOVE
+0230; C; 0231; # LATIN CAPITAL LETTER O WITH DOT ABOVE AND MACRON
+0232; C; 0233; # LATIN CAPITAL LETTER Y WITH MACRON
+023A; C; 2C65; # LATIN CAPITAL LETTER A WITH STROKE
+023B; C; 023C; # LATIN CAPITAL LETTER C WITH STROKE
+023D; C; 019A; # LATIN CAPITAL LETTER L WITH BAR
+023E; C; 2C66; # LATIN CAPITAL LETTER T WITH DIAGONAL STROKE
+0241; C; 0242; # LATIN CAPITAL LETTER GLOTTAL STOP
+0243; C; 0180; # LATIN CAPITAL LETTER B WITH STROKE
+0244; C; 0289; # LATIN CAPITAL LETTER U BAR
+0245; C; 028C; # LATIN CAPITAL LETTER TURNED V
+0246; C; 0247; # LATIN CAPITAL LETTER E WITH STROKE
+0248; C; 0249; # LATIN CAPITAL LETTER J WITH STROKE
+024A; C; 024B; # LATIN CAPITAL LETTER SMALL Q WITH HOOK TAIL
+024C; C; 024D; # LATIN CAPITAL LETTER R WITH STROKE
+024E; C; 024F; # LATIN CAPITAL LETTER Y WITH STROKE
+0345; C; 03B9; # COMBINING GREEK YPOGEGRAMMENI
+0370; C; 0371; # GREEK CAPITAL LETTER HETA
+0372; C; 0373; # GREEK CAPITAL LETTER ARCHAIC SAMPI
+0376; C; 0377; # GREEK CAPITAL LETTER PAMPHYLIAN DIGAMMA
+037F; C; 03F3; # GREEK CAPITAL LETTER YOT
+0386; C; 03AC; # GREEK CAPITAL LETTER ALPHA WITH TONOS
+0388; C; 03AD; # GREEK CAPITAL LETTER EPSILON WITH TONOS
+0389; C; 03AE; # GREEK CAPITAL LETTER ETA WITH TONOS
+038A; C; 03AF; # GREEK CAPITAL LETTER IOTA WITH TONOS
+038C; C; 03CC; # GREEK CAPITAL LETTER OMICRON WITH TONOS
+038E; C; 03CD; # GREEK CAPITAL LETTER UPSILON WITH TONOS
+038F; C; 03CE; # GREEK CAPITAL LETTER OMEGA WITH TONOS
+0390; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND TONOS
+0391; C; 03B1; # GREEK CAPITAL LETTER ALPHA
+0392; C; 03B2; # GREEK CAPITAL LETTER BETA
+0393; C; 03B3; # GREEK CAPITAL LETTER GAMMA
+0394; C; 03B4; # GREEK CAPITAL LETTER DELTA
+0395; C; 03B5; # GREEK CAPITAL LETTER EPSILON
+0396; C; 03B6; # GREEK CAPITAL LETTER ZETA
+0397; C; 03B7; # GREEK CAPITAL LETTER ETA
+0398; C; 03B8; # GREEK CAPITAL LETTER THETA
+0399; C; 03B9; # GREEK CAPITAL LETTER IOTA
+039A; C; 03BA; # GREEK CAPITAL LETTER KAPPA
+039B; C; 03BB; # GREEK CAPITAL LETTER LAMDA
+039C; C; 03BC; # GREEK CAPITAL LETTER MU
+039D; C; 03BD; # GREEK CAPITAL LETTER NU
+039E; C; 03BE; # GREEK CAPITAL LETTER XI
+039F; C; 03BF; # GREEK CAPITAL LETTER OMICRON
+03A0; C; 03C0; # GREEK CAPITAL LETTER PI
+03A1; C; 03C1; # GREEK CAPITAL LETTER RHO
+03A3; C; 03C3; # GREEK CAPITAL LETTER SIGMA
+03A4; C; 03C4; # GREEK CAPITAL LETTER TAU
+03A5; C; 03C5; # GREEK CAPITAL LETTER UPSILON
+03A6; C; 03C6; # GREEK CAPITAL LETTER PHI
+03A7; C; 03C7; # GREEK CAPITAL LETTER CHI
+03A8; C; 03C8; # GREEK CAPITAL LETTER PSI
+03A9; C; 03C9; # GREEK CAPITAL LETTER OMEGA
+03AA; C; 03CA; # GREEK CAPITAL LETTER IOTA WITH DIALYTIKA
+03AB; C; 03CB; # GREEK CAPITAL LETTER UPSILON WITH DIALYTIKA
+03B0; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND TONOS
+03C2; C; 03C3; # GREEK SMALL LETTER FINAL SIGMA
+03CF; C; 03D7; # GREEK CAPITAL KAI SYMBOL
+03D0; C; 03B2; # GREEK BETA SYMBOL
+03D1; C; 03B8; # GREEK THETA SYMBOL
+03D5; C; 03C6; # GREEK PHI SYMBOL
+03D6; C; 03C0; # GREEK PI SYMBOL
+03D8; C; 03D9; # GREEK LETTER ARCHAIC KOPPA
+03DA; C; 03DB; # GREEK LETTER STIGMA
+03DC; C; 03DD; # GREEK LETTER DIGAMMA
+03DE; C; 03DF; # GREEK LETTER KOPPA
+03E0; C; 03E1; # GREEK LETTER SAMPI
+03E2; C; 03E3; # COPTIC CAPITAL LETTER SHEI
+03E4; C; 03E5; # COPTIC CAPITAL LETTER FEI
+03E6; C; 03E7; # COPTIC CAPITAL LETTER KHEI
+03E8; C; 03E9; # COPTIC CAPITAL LETTER HORI
+03EA; C; 03EB; # COPTIC CAPITAL LETTER GANGIA
+03EC; C; 03ED; # COPTIC CAPITAL LETTER SHIMA
+03EE; C; 03EF; # COPTIC CAPITAL LETTER DEI
+03F0; C; 03BA; # GREEK KAPPA SYMBOL
+03F1; C; 03C1; # GREEK RHO SYMBOL
+03F4; C; 03B8; # GREEK CAPITAL THETA SYMBOL
+03F5; C; 03B5; # GREEK LUNATE EPSILON SYMBOL
+03F7; C; 03F8; # GREEK CAPITAL LETTER SHO
+03F9; C; 03F2; # GREEK CAPITAL LUNATE SIGMA SYMBOL
+03FA; C; 03FB; # GREEK CAPITAL LETTER SAN
+03FD; C; 037B; # GREEK CAPITAL REVERSED LUNATE SIGMA SYMBOL
+03FE; C; 037C; # GREEK CAPITAL DOTTED LUNATE SIGMA SYMBOL
+03FF; C; 037D; # GREEK CAPITAL REVERSED DOTTED LUNATE SIGMA SYMBOL
+0400; C; 0450; # CYRILLIC CAPITAL LETTER IE WITH GRAVE
+0401; C; 0451; # CYRILLIC CAPITAL LETTER IO
+0402; C; 0452; # CYRILLIC CAPITAL LETTER DJE
+0403; C; 0453; # CYRILLIC CAPITAL LETTER GJE
+0404; C; 0454; # CYRILLIC CAPITAL LETTER UKRAINIAN IE
+0405; C; 0455; # CYRILLIC CAPITAL LETTER DZE
+0406; C; 0456; # CYRILLIC CAPITAL LETTER BYELORUSSIAN-UKRAINIAN I
+0407; C; 0457; # CYRILLIC CAPITAL LETTER YI
+0408; C; 0458; # CYRILLIC CAPITAL LETTER JE
+0409; C; 0459; # CYRILLIC CAPITAL LETTER LJE
+040A; C; 045A; # CYRILLIC CAPITAL LETTER NJE
+040B; C; 045B; # CYRILLIC CAPITAL LETTER TSHE
+040C; C; 045C; # CYRILLIC CAPITAL LETTER KJE
+040D; C; 045D; # CYRILLIC CAPITAL LETTER I WITH GRAVE
+040E; C; 045E; # CYRILLIC CAPITAL LETTER SHORT U
+040F; C; 045F; # CYRILLIC CAPITAL LETTER DZHE
+0410; C; 0430; # CYRILLIC CAPITAL LETTER A
+0411; C; 0431; # CYRILLIC CAPITAL LETTER BE
+0412; C; 0432; # CYRILLIC CAPITAL LETTER VE
+0413; C; 0433; # CYRILLIC CAPITAL LETTER GHE
+0414; C; 0434; # CYRILLIC CAPITAL LETTER DE
+0415; C; 0435; # CYRILLIC CAPITAL LETTER IE
+0416; C; 0436; # CYRILLIC CAPITAL LETTER ZHE
+0417; C; 0437; # CYRILLIC CAPITAL LETTER ZE
+0418; C; 0438; # CYRILLIC CAPITAL LETTER I
+0419; C; 0439; # CYRILLIC CAPITAL LETTER SHORT I
+041A; C; 043A; # CYRILLIC CAPITAL LETTER KA
+041B; C; 043B; # CYRILLIC CAPITAL LETTER EL
+041C; C; 043C; # CYRILLIC CAPITAL LETTER EM
+041D; C; 043D; # CYRILLIC CAPITAL LETTER EN
+041E; C; 043E; # CYRILLIC CAPITAL LETTER O
+041F; C; 043F; # CYRILLIC CAPITAL LETTER PE
+0420; C; 0440; # CYRILLIC CAPITAL LETTER ER
+0421; C; 0441; # CYRILLIC CAPITAL LETTER ES
+0422; C; 0442; # CYRILLIC CAPITAL LETTER TE
+0423; C; 0443; # CYRILLIC CAPITAL LETTER U
+0424; C; 0444; # CYRILLIC CAPITAL LETTER EF
+0425; C; 0445; # CYRILLIC CAPITAL LETTER HA
+0426; C; 0446; # CYRILLIC CAPITAL LETTER TSE
+0427; C; 0447; # CYRILLIC CAPITAL LETTER CHE
+0428; C; 0448; # CYRILLIC CAPITAL LETTER SHA
+0429; C; 0449; # CYRILLIC CAPITAL LETTER SHCHA
+042A; C; 044A; # CYRILLIC CAPITAL LETTER HARD SIGN
+042B; C; 044B; # CYRILLIC CAPITAL LETTER YERU
+042C; C; 044C; # CYRILLIC CAPITAL LETTER SOFT SIGN
+042D; C; 044D; # CYRILLIC CAPITAL LETTER E
+042E; C; 044E; # CYRILLIC CAPITAL LETTER YU
+042F; C; 044F; # CYRILLIC CAPITAL LETTER YA
+0460; C; 0461; # CYRILLIC CAPITAL LETTER OMEGA
+0462; C; 0463; # CYRILLIC CAPITAL LETTER YAT
+0464; C; 0465; # CYRILLIC CAPITAL LETTER IOTIFIED E
+0466; C; 0467; # CYRILLIC CAPITAL LETTER LITTLE YUS
+0468; C; 0469; # CYRILLIC CAPITAL LETTER IOTIFIED LITTLE YUS
+046A; C; 046B; # CYRILLIC CAPITAL LETTER BIG YUS
+046C; C; 046D; # CYRILLIC CAPITAL LETTER IOTIFIED BIG YUS
+046E; C; 046F; # CYRILLIC CAPITAL LETTER KSI
+0470; C; 0471; # CYRILLIC CAPITAL LETTER PSI
+0472; C; 0473; # CYRILLIC CAPITAL LETTER FITA
+0474; C; 0475; # CYRILLIC CAPITAL LETTER IZHITSA
+0476; C; 0477; # CYRILLIC CAPITAL LETTER IZHITSA WITH DOUBLE GRAVE ACCENT
+0478; C; 0479; # CYRILLIC CAPITAL LETTER UK
+047A; C; 047B; # CYRILLIC CAPITAL LETTER ROUND OMEGA
+047C; C; 047D; # CYRILLIC CAPITAL LETTER OMEGA WITH TITLO
+047E; C; 047F; # CYRILLIC CAPITAL LETTER OT
+0480; C; 0481; # CYRILLIC CAPITAL LETTER KOPPA
+048A; C; 048B; # CYRILLIC CAPITAL LETTER SHORT I WITH TAIL
+048C; C; 048D; # CYRILLIC CAPITAL LETTER SEMISOFT SIGN
+048E; C; 048F; # CYRILLIC CAPITAL LETTER ER WITH TICK
+0490; C; 0491; # CYRILLIC CAPITAL LETTER GHE WITH UPTURN
+0492; C; 0493; # CYRILLIC CAPITAL LETTER GHE WITH STROKE
+0494; C; 0495; # CYRILLIC CAPITAL LETTER GHE WITH MIDDLE HOOK
+0496; C; 0497; # CYRILLIC CAPITAL LETTER ZHE WITH DESCENDER
+0498; C; 0499; # CYRILLIC CAPITAL LETTER ZE WITH DESCENDER
+049A; C; 049B; # CYRILLIC CAPITAL LETTER KA WITH DESCENDER
+049C; C; 049D; # CYRILLIC CAPITAL LETTER KA WITH VERTICAL STROKE
+049E; C; 049F; # CYRILLIC CAPITAL LETTER KA WITH STROKE
+04A0; C; 04A1; # CYRILLIC CAPITAL LETTER BASHKIR KA
+04A2; C; 04A3; # CYRILLIC CAPITAL LETTER EN WITH DESCENDER
+04A4; C; 04A5; # CYRILLIC CAPITAL LIGATURE EN GHE
+04A6; C; 04A7; # CYRILLIC CAPITAL LETTER PE WITH MIDDLE HOOK
+04A8; C; 04A9; # CYRILLIC CAPITAL LETTER ABKHASIAN HA
+04AA; C; 04AB; # CYRILLIC CAPITAL LETTER ES WITH DESCENDER
+04AC; C; 04AD; # CYRILLIC CAPITAL LETTER TE WITH DESCENDER
+04AE; C; 04AF; # CYRILLIC CAPITAL LETTER STRAIGHT U
+04B0; C; 04B1; # CYRILLIC CAPITAL LETTER STRAIGHT U WITH STROKE
+04B2; C; 04B3; # CYRILLIC CAPITAL LETTER HA WITH DESCENDER
+04B4; C; 04B5; # CYRILLIC CAPITAL LIGATURE TE TSE
+04B6; C; 04B7; # CYRILLIC CAPITAL LETTER CHE WITH DESCENDER
+04B8; C; 04B9; # CYRILLIC CAPITAL LETTER CHE WITH VERTICAL STROKE
+04BA; C; 04BB; # CYRILLIC CAPITAL LETTER SHHA
+04BC; C; 04BD; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE
+04BE; C; 04BF; # CYRILLIC CAPITAL LETTER ABKHASIAN CHE WITH DESCENDER
+04C0; C; 04CF; # CYRILLIC LETTER PALOCHKA
+04C1; C; 04C2; # CYRILLIC CAPITAL LETTER ZHE WITH BREVE
+04C3; C; 04C4; # CYRILLIC CAPITAL LETTER KA WITH HOOK
+04C5; C; 04C6; # CYRILLIC CAPITAL LETTER EL WITH TAIL
+04C7; C; 04C8; # CYRILLIC CAPITAL LETTER EN WITH HOOK
+04C9; C; 04CA; # CYRILLIC CAPITAL LETTER EN WITH TAIL
+04CB; C; 04CC; # CYRILLIC CAPITAL LETTER KHAKASSIAN CHE
+04CD; C; 04CE; # CYRILLIC CAPITAL LETTER EM WITH TAIL
+04D0; C; 04D1; # CYRILLIC CAPITAL LETTER A WITH BREVE
+04D2; C; 04D3; # CYRILLIC CAPITAL LETTER A WITH DIAERESIS
+04D4; C; 04D5; # CYRILLIC CAPITAL LIGATURE A IE
+04D6; C; 04D7; # CYRILLIC CAPITAL LETTER IE WITH BREVE
+04D8; C; 04D9; # CYRILLIC CAPITAL LETTER SCHWA
+04DA; C; 04DB; # CYRILLIC CAPITAL LETTER SCHWA WITH DIAERESIS
+04DC; C; 04DD; # CYRILLIC CAPITAL LETTER ZHE WITH DIAERESIS
+04DE; C; 04DF; # CYRILLIC CAPITAL LETTER ZE WITH DIAERESIS
+04E0; C; 04E1; # CYRILLIC CAPITAL LETTER ABKHASIAN DZE
+04E2; C; 04E3; # CYRILLIC CAPITAL LETTER I WITH MACRON
+04E4; C; 04E5; # CYRILLIC CAPITAL LETTER I WITH DIAERESIS
+04E6; C; 04E7; # CYRILLIC CAPITAL LETTER O WITH DIAERESIS
+04E8; C; 04E9; # CYRILLIC CAPITAL LETTER BARRED O
+04EA; C; 04EB; # CYRILLIC CAPITAL LETTER BARRED O WITH DIAERESIS
+04EC; C; 04ED; # CYRILLIC CAPITAL LETTER E WITH DIAERESIS
+04EE; C; 04EF; # CYRILLIC CAPITAL LETTER U WITH MACRON
+04F0; C; 04F1; # CYRILLIC CAPITAL LETTER U WITH DIAERESIS
+04F2; C; 04F3; # CYRILLIC CAPITAL LETTER U WITH DOUBLE ACUTE
+04F4; C; 04F5; # CYRILLIC CAPITAL LETTER CHE WITH DIAERESIS
+04F6; C; 04F7; # CYRILLIC CAPITAL LETTER GHE WITH DESCENDER
+04F8; C; 04F9; # CYRILLIC CAPITAL LETTER YERU WITH DIAERESIS
+04FA; C; 04FB; # CYRILLIC CAPITAL LETTER GHE WITH STROKE AND HOOK
+04FC; C; 04FD; # CYRILLIC CAPITAL LETTER HA WITH HOOK
+04FE; C; 04FF; # CYRILLIC CAPITAL LETTER HA WITH STROKE
+0500; C; 0501; # CYRILLIC CAPITAL LETTER KOMI DE
+0502; C; 0503; # CYRILLIC CAPITAL LETTER KOMI DJE
+0504; C; 0505; # CYRILLIC CAPITAL LETTER KOMI ZJE
+0506; C; 0507; # CYRILLIC CAPITAL LETTER KOMI DZJE
+0508; C; 0509; # CYRILLIC CAPITAL LETTER KOMI LJE
+050A; C; 050B; # CYRILLIC CAPITAL LETTER KOMI NJE
+050C; C; 050D; # CYRILLIC CAPITAL LETTER KOMI SJE
+050E; C; 050F; # CYRILLIC CAPITAL LETTER KOMI TJE
+0510; C; 0511; # CYRILLIC CAPITAL LETTER REVERSED ZE
+0512; C; 0513; # CYRILLIC CAPITAL LETTER EL WITH HOOK
+0514; C; 0515; # CYRILLIC CAPITAL LETTER LHA
+0516; C; 0517; # CYRILLIC CAPITAL LETTER RHA
+0518; C; 0519; # CYRILLIC CAPITAL LETTER YAE
+051A; C; 051B; # CYRILLIC CAPITAL LETTER QA
+051C; C; 051D; # CYRILLIC CAPITAL LETTER WE
+051E; C; 051F; # CYRILLIC CAPITAL LETTER ALEUT KA
+0520; C; 0521; # CYRILLIC CAPITAL LETTER EL WITH MIDDLE HOOK
+0522; C; 0523; # CYRILLIC CAPITAL LETTER EN WITH MIDDLE HOOK
+0524; C; 0525; # CYRILLIC CAPITAL LETTER PE WITH DESCENDER
+0526; C; 0527; # CYRILLIC CAPITAL LETTER SHHA WITH DESCENDER
+0528; C; 0529; # CYRILLIC CAPITAL LETTER EN WITH LEFT HOOK
+052A; C; 052B; # CYRILLIC CAPITAL LETTER DZZHE
+052C; C; 052D; # CYRILLIC CAPITAL LETTER DCHE
+052E; C; 052F; # CYRILLIC CAPITAL LETTER EL WITH DESCENDER
+0531; C; 0561; # ARMENIAN CAPITAL LETTER AYB
+0532; C; 0562; # ARMENIAN CAPITAL LETTER BEN
+0533; C; 0563; # ARMENIAN CAPITAL LETTER GIM
+0534; C; 0564; # ARMENIAN CAPITAL LETTER DA
+0535; C; 0565; # ARMENIAN CAPITAL LETTER ECH
+0536; C; 0566; # ARMENIAN CAPITAL LETTER ZA
+0537; C; 0567; # ARMENIAN CAPITAL LETTER EH
+0538; C; 0568; # ARMENIAN CAPITAL LETTER ET
+0539; C; 0569; # ARMENIAN CAPITAL LETTER TO
+053A; C; 056A; # ARMENIAN CAPITAL LETTER ZHE
+053B; C; 056B; # ARMENIAN CAPITAL LETTER INI
+053C; C; 056C; # ARMENIAN CAPITAL LETTER LIWN
+053D; C; 056D; # ARMENIAN CAPITAL LETTER XEH
+053E; C; 056E; # ARMENIAN CAPITAL LETTER CA
+053F; C; 056F; # ARMENIAN CAPITAL LETTER KEN
+0540; C; 0570; # ARMENIAN CAPITAL LETTER HO
+0541; C; 0571; # ARMENIAN CAPITAL LETTER JA
+0542; C; 0572; # ARMENIAN CAPITAL LETTER GHAD
+0543; C; 0573; # ARMENIAN CAPITAL LETTER CHEH
+0544; C; 0574; # ARMENIAN CAPITAL LETTER MEN
+0545; C; 0575; # ARMENIAN CAPITAL LETTER YI
+0546; C; 0576; # ARMENIAN CAPITAL LETTER NOW
+0547; C; 0577; # ARMENIAN CAPITAL LETTER SHA
+0548; C; 0578; # ARMENIAN CAPITAL LETTER VO
+0549; C; 0579; # ARMENIAN CAPITAL LETTER CHA
+054A; C; 057A; # ARMENIAN CAPITAL LETTER PEH
+054B; C; 057B; # ARMENIAN CAPITAL LETTER JHEH
+054C; C; 057C; # ARMENIAN CAPITAL LETTER RA
+054D; C; 057D; # ARMENIAN CAPITAL LETTER SEH
+054E; C; 057E; # ARMENIAN CAPITAL LETTER VEW
+054F; C; 057F; # ARMENIAN CAPITAL LETTER TIWN
+0550; C; 0580; # ARMENIAN CAPITAL LETTER REH
+0551; C; 0581; # ARMENIAN CAPITAL LETTER CO
+0552; C; 0582; # ARMENIAN CAPITAL LETTER YIWN
+0553; C; 0583; # ARMENIAN CAPITAL LETTER PIWR
+0554; C; 0584; # ARMENIAN CAPITAL LETTER KEH
+0555; C; 0585; # ARMENIAN CAPITAL LETTER OH
+0556; C; 0586; # ARMENIAN CAPITAL LETTER FEH
+0587; F; 0565 0582; # ARMENIAN SMALL LIGATURE ECH YIWN
+10A0; C; 2D00; # GEORGIAN CAPITAL LETTER AN
+10A1; C; 2D01; # GEORGIAN CAPITAL LETTER BAN
+10A2; C; 2D02; # GEORGIAN CAPITAL LETTER GAN
+10A3; C; 2D03; # GEORGIAN CAPITAL LETTER DON
+10A4; C; 2D04; # GEORGIAN CAPITAL LETTER EN
+10A5; C; 2D05; # GEORGIAN CAPITAL LETTER VIN
+10A6; C; 2D06; # GEORGIAN CAPITAL LETTER ZEN
+10A7; C; 2D07; # GEORGIAN CAPITAL LETTER TAN
+10A8; C; 2D08; # GEORGIAN CAPITAL LETTER IN
+10A9; C; 2D09; # GEORGIAN CAPITAL LETTER KAN
+10AA; C; 2D0A; # GEORGIAN CAPITAL LETTER LAS
+10AB; C; 2D0B; # GEORGIAN CAPITAL LETTER MAN
+10AC; C; 2D0C; # GEORGIAN CAPITAL LETTER NAR
+10AD; C; 2D0D; # GEORGIAN CAPITAL LETTER ON
+10AE; C; 2D0E; # GEORGIAN CAPITAL LETTER PAR
+10AF; C; 2D0F; # GEORGIAN CAPITAL LETTER ZHAR
+10B0; C; 2D10; # GEORGIAN CAPITAL LETTER RAE
+10B1; C; 2D11; # GEORGIAN CAPITAL LETTER SAN
+10B2; C; 2D12; # GEORGIAN CAPITAL LETTER TAR
+10B3; C; 2D13; # GEORGIAN CAPITAL LETTER UN
+10B4; C; 2D14; # GEORGIAN CAPITAL LETTER PHAR
+10B5; C; 2D15; # GEORGIAN CAPITAL LETTER KHAR
+10B6; C; 2D16; # GEORGIAN CAPITAL LETTER GHAN
+10B7; C; 2D17; # GEORGIAN CAPITAL LETTER QAR
+10B8; C; 2D18; # GEORGIAN CAPITAL LETTER SHIN
+10B9; C; 2D19; # GEORGIAN CAPITAL LETTER CHIN
+10BA; C; 2D1A; # GEORGIAN CAPITAL LETTER CAN
+10BB; C; 2D1B; # GEORGIAN CAPITAL LETTER JIL
+10BC; C; 2D1C; # GEORGIAN CAPITAL LETTER CIL
+10BD; C; 2D1D; # GEORGIAN CAPITAL LETTER CHAR
+10BE; C; 2D1E; # GEORGIAN CAPITAL LETTER XAN
+10BF; C; 2D1F; # GEORGIAN CAPITAL LETTER JHAN
+10C0; C; 2D20; # GEORGIAN CAPITAL LETTER HAE
+10C1; C; 2D21; # GEORGIAN CAPITAL LETTER HE
+10C2; C; 2D22; # GEORGIAN CAPITAL LETTER HIE
+10C3; C; 2D23; # GEORGIAN CAPITAL LETTER WE
+10C4; C; 2D24; # GEORGIAN CAPITAL LETTER HAR
+10C5; C; 2D25; # GEORGIAN CAPITAL LETTER HOE
+10C7; C; 2D27; # GEORGIAN CAPITAL LETTER YN
+10CD; C; 2D2D; # GEORGIAN CAPITAL LETTER AEN
+13F8; C; 13F0; # CHEROKEE SMALL LETTER YE
+13F9; C; 13F1; # CHEROKEE SMALL LETTER YI
+13FA; C; 13F2; # CHEROKEE SMALL LETTER YO
+13FB; C; 13F3; # CHEROKEE SMALL LETTER YU
+13FC; C; 13F4; # CHEROKEE SMALL LETTER YV
+13FD; C; 13F5; # CHEROKEE SMALL LETTER MV
+1C80; C; 0432; # CYRILLIC SMALL LETTER ROUNDED VE
+1C81; C; 0434; # CYRILLIC SMALL LETTER LONG-LEGGED DE
+1C82; C; 043E; # CYRILLIC SMALL LETTER NARROW O
+1C83; C; 0441; # CYRILLIC SMALL LETTER WIDE ES
+1C84; C; 0442; # CYRILLIC SMALL LETTER TALL TE
+1C85; C; 0442; # CYRILLIC SMALL LETTER THREE-LEGGED TE
+1C86; C; 044A; # CYRILLIC SMALL LETTER TALL HARD SIGN
+1C87; C; 0463; # CYRILLIC SMALL LETTER TALL YAT
+1C88; C; A64B; # CYRILLIC SMALL LETTER UNBLENDED UK
+1C90; C; 10D0; # GEORGIAN MTAVRULI CAPITAL LETTER AN
+1C91; C; 10D1; # GEORGIAN MTAVRULI CAPITAL LETTER BAN
+1C92; C; 10D2; # GEORGIAN MTAVRULI CAPITAL LETTER GAN
+1C93; C; 10D3; # GEORGIAN MTAVRULI CAPITAL LETTER DON
+1C94; C; 10D4; # GEORGIAN MTAVRULI CAPITAL LETTER EN
+1C95; C; 10D5; # GEORGIAN MTAVRULI CAPITAL LETTER VIN
+1C96; C; 10D6; # GEORGIAN MTAVRULI CAPITAL LETTER ZEN
+1C97; C; 10D7; # GEORGIAN MTAVRULI CAPITAL LETTER TAN
+1C98; C; 10D8; # GEORGIAN MTAVRULI CAPITAL LETTER IN
+1C99; C; 10D9; # GEORGIAN MTAVRULI CAPITAL LETTER KAN
+1C9A; C; 10DA; # GEORGIAN MTAVRULI CAPITAL LETTER LAS
+1C9B; C; 10DB; # GEORGIAN MTAVRULI CAPITAL LETTER MAN
+1C9C; C; 10DC; # GEORGIAN MTAVRULI CAPITAL LETTER NAR
+1C9D; C; 10DD; # GEORGIAN MTAVRULI CAPITAL LETTER ON
+1C9E; C; 10DE; # GEORGIAN MTAVRULI CAPITAL LETTER PAR
+1C9F; C; 10DF; # GEORGIAN MTAVRULI CAPITAL LETTER ZHAR
+1CA0; C; 10E0; # GEORGIAN MTAVRULI CAPITAL LETTER RAE
+1CA1; C; 10E1; # GEORGIAN MTAVRULI CAPITAL LETTER SAN
+1CA2; C; 10E2; # GEORGIAN MTAVRULI CAPITAL LETTER TAR
+1CA3; C; 10E3; # GEORGIAN MTAVRULI CAPITAL LETTER UN
+1CA4; C; 10E4; # GEORGIAN MTAVRULI CAPITAL LETTER PHAR
+1CA5; C; 10E5; # GEORGIAN MTAVRULI CAPITAL LETTER KHAR
+1CA6; C; 10E6; # GEORGIAN MTAVRULI CAPITAL LETTER GHAN
+1CA7; C; 10E7; # GEORGIAN MTAVRULI CAPITAL LETTER QAR
+1CA8; C; 10E8; # GEORGIAN MTAVRULI CAPITAL LETTER SHIN
+1CA9; C; 10E9; # GEORGIAN MTAVRULI CAPITAL LETTER CHIN
+1CAA; C; 10EA; # GEORGIAN MTAVRULI CAPITAL LETTER CAN
+1CAB; C; 10EB; # GEORGIAN MTAVRULI CAPITAL LETTER JIL
+1CAC; C; 10EC; # GEORGIAN MTAVRULI CAPITAL LETTER CIL
+1CAD; C; 10ED; # GEORGIAN MTAVRULI CAPITAL LETTER CHAR
+1CAE; C; 10EE; # GEORGIAN MTAVRULI CAPITAL LETTER XAN
+1CAF; C; 10EF; # GEORGIAN MTAVRULI CAPITAL LETTER JHAN
+1CB0; C; 10F0; # GEORGIAN MTAVRULI CAPITAL LETTER HAE
+1CB1; C; 10F1; # GEORGIAN MTAVRULI CAPITAL LETTER HE
+1CB2; C; 10F2; # GEORGIAN MTAVRULI CAPITAL LETTER HIE
+1CB3; C; 10F3; # GEORGIAN MTAVRULI CAPITAL LETTER WE
+1CB4; C; 10F4; # GEORGIAN MTAVRULI CAPITAL LETTER HAR
+1CB5; C; 10F5; # GEORGIAN MTAVRULI CAPITAL LETTER HOE
+1CB6; C; 10F6; # GEORGIAN MTAVRULI CAPITAL LETTER FI
+1CB7; C; 10F7; # GEORGIAN MTAVRULI CAPITAL LETTER YN
+1CB8; C; 10F8; # GEORGIAN MTAVRULI CAPITAL LETTER ELIFI
+1CB9; C; 10F9; # GEORGIAN MTAVRULI CAPITAL LETTER TURNED GAN
+1CBA; C; 10FA; # GEORGIAN MTAVRULI CAPITAL LETTER AIN
+1CBD; C; 10FD; # GEORGIAN MTAVRULI CAPITAL LETTER AEN
+1CBE; C; 10FE; # GEORGIAN MTAVRULI CAPITAL LETTER HARD SIGN
+1CBF; C; 10FF; # GEORGIAN MTAVRULI CAPITAL LETTER LABIAL SIGN
+1E00; C; 1E01; # LATIN CAPITAL LETTER A WITH RING BELOW
+1E02; C; 1E03; # LATIN CAPITAL LETTER B WITH DOT ABOVE
+1E04; C; 1E05; # LATIN CAPITAL LETTER B WITH DOT BELOW
+1E06; C; 1E07; # LATIN CAPITAL LETTER B WITH LINE BELOW
+1E08; C; 1E09; # LATIN CAPITAL LETTER C WITH CEDILLA AND ACUTE
+1E0A; C; 1E0B; # LATIN CAPITAL LETTER D WITH DOT ABOVE
+1E0C; C; 1E0D; # LATIN CAPITAL LETTER D WITH DOT BELOW
+1E0E; C; 1E0F; # LATIN CAPITAL LETTER D WITH LINE BELOW
+1E10; C; 1E11; # LATIN CAPITAL LETTER D WITH CEDILLA
+1E12; C; 1E13; # LATIN CAPITAL LETTER D WITH CIRCUMFLEX BELOW
+1E14; C; 1E15; # LATIN CAPITAL LETTER E WITH MACRON AND GRAVE
+1E16; C; 1E17; # LATIN CAPITAL LETTER E WITH MACRON AND ACUTE
+1E18; C; 1E19; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX BELOW
+1E1A; C; 1E1B; # LATIN CAPITAL LETTER E WITH TILDE BELOW
+1E1C; C; 1E1D; # LATIN CAPITAL LETTER E WITH CEDILLA AND BREVE
+1E1E; C; 1E1F; # LATIN CAPITAL LETTER F WITH DOT ABOVE
+1E20; C; 1E21; # LATIN CAPITAL LETTER G WITH MACRON
+1E22; C; 1E23; # LATIN CAPITAL LETTER H WITH DOT ABOVE
+1E24; C; 1E25; # LATIN CAPITAL LETTER H WITH DOT BELOW
+1E26; C; 1E27; # LATIN CAPITAL LETTER H WITH DIAERESIS
+1E28; C; 1E29; # LATIN CAPITAL LETTER H WITH CEDILLA
+1E2A; C; 1E2B; # LATIN CAPITAL LETTER H WITH BREVE BELOW
+1E2C; C; 1E2D; # LATIN CAPITAL LETTER I WITH TILDE BELOW
+1E2E; C; 1E2F; # LATIN CAPITAL LETTER I WITH DIAERESIS AND ACUTE
+1E30; C; 1E31; # LATIN CAPITAL LETTER K WITH ACUTE
+1E32; C; 1E33; # LATIN CAPITAL LETTER K WITH DOT BELOW
+1E34; C; 1E35; # LATIN CAPITAL LETTER K WITH LINE BELOW
+1E36; C; 1E37; # LATIN CAPITAL LETTER L WITH DOT BELOW
+1E38; C; 1E39; # LATIN CAPITAL LETTER L WITH DOT BELOW AND MACRON
+1E3A; C; 1E3B; # LATIN CAPITAL LETTER L WITH LINE BELOW
+1E3C; C; 1E3D; # LATIN CAPITAL LETTER L WITH CIRCUMFLEX BELOW
+1E3E; C; 1E3F; # LATIN CAPITAL LETTER M WITH ACUTE
+1E40; C; 1E41; # LATIN CAPITAL LETTER M WITH DOT ABOVE
+1E42; C; 1E43; # LATIN CAPITAL LETTER M WITH DOT BELOW
+1E44; C; 1E45; # LATIN CAPITAL LETTER N WITH DOT ABOVE
+1E46; C; 1E47; # LATIN CAPITAL LETTER N WITH DOT BELOW
+1E48; C; 1E49; # LATIN CAPITAL LETTER N WITH LINE BELOW
+1E4A; C; 1E4B; # LATIN CAPITAL LETTER N WITH CIRCUMFLEX BELOW
+1E4C; C; 1E4D; # LATIN CAPITAL LETTER O WITH TILDE AND ACUTE
+1E4E; C; 1E4F; # LATIN CAPITAL LETTER O WITH TILDE AND DIAERESIS
+1E50; C; 1E51; # LATIN CAPITAL LETTER O WITH MACRON AND GRAVE
+1E52; C; 1E53; # LATIN CAPITAL LETTER O WITH MACRON AND ACUTE
+1E54; C; 1E55; # LATIN CAPITAL LETTER P WITH ACUTE
+1E56; C; 1E57; # LATIN CAPITAL LETTER P WITH DOT ABOVE
+1E58; C; 1E59; # LATIN CAPITAL LETTER R WITH DOT ABOVE
+1E5A; C; 1E5B; # LATIN CAPITAL LETTER R WITH DOT BELOW
+1E5C; C; 1E5D; # LATIN CAPITAL LETTER R WITH DOT BELOW AND MACRON
+1E5E; C; 1E5F; # LATIN CAPITAL LETTER R WITH LINE BELOW
+1E60; C; 1E61; # LATIN CAPITAL LETTER S WITH DOT ABOVE
+1E62; C; 1E63; # LATIN CAPITAL LETTER S WITH DOT BELOW
+1E64; C; 1E65; # LATIN CAPITAL LETTER S WITH ACUTE AND DOT ABOVE
+1E66; C; 1E67; # LATIN CAPITAL LETTER S WITH CARON AND DOT ABOVE
+1E68; C; 1E69; # LATIN CAPITAL LETTER S WITH DOT BELOW AND DOT ABOVE
+1E6A; C; 1E6B; # LATIN CAPITAL LETTER T WITH DOT ABOVE
+1E6C; C; 1E6D; # LATIN CAPITAL LETTER T WITH DOT BELOW
+1E6E; C; 1E6F; # LATIN CAPITAL LETTER T WITH LINE BELOW
+1E70; C; 1E71; # LATIN CAPITAL LETTER T WITH CIRCUMFLEX BELOW
+1E72; C; 1E73; # LATIN CAPITAL LETTER U WITH DIAERESIS BELOW
+1E74; C; 1E75; # LATIN CAPITAL LETTER U WITH TILDE BELOW
+1E76; C; 1E77; # LATIN CAPITAL LETTER U WITH CIRCUMFLEX BELOW
+1E78; C; 1E79; # LATIN CAPITAL LETTER U WITH TILDE AND ACUTE
+1E7A; C; 1E7B; # LATIN CAPITAL LETTER U WITH MACRON AND DIAERESIS
+1E7C; C; 1E7D; # LATIN CAPITAL LETTER V WITH TILDE
+1E7E; C; 1E7F; # LATIN CAPITAL LETTER V WITH DOT BELOW
+1E80; C; 1E81; # LATIN CAPITAL LETTER W WITH GRAVE
+1E82; C; 1E83; # LATIN CAPITAL LETTER W WITH ACUTE
+1E84; C; 1E85; # LATIN CAPITAL LETTER W WITH DIAERESIS
+1E86; C; 1E87; # LATIN CAPITAL LETTER W WITH DOT ABOVE
+1E88; C; 1E89; # LATIN CAPITAL LETTER W WITH DOT BELOW
+1E8A; C; 1E8B; # LATIN CAPITAL LETTER X WITH DOT ABOVE
+1E8C; C; 1E8D; # LATIN CAPITAL LETTER X WITH DIAERESIS
+1E8E; C; 1E8F; # LATIN CAPITAL LETTER Y WITH DOT ABOVE
+1E90; C; 1E91; # LATIN CAPITAL LETTER Z WITH CIRCUMFLEX
+1E92; C; 1E93; # LATIN CAPITAL LETTER Z WITH DOT BELOW
+1E94; C; 1E95; # LATIN CAPITAL LETTER Z WITH LINE BELOW
+1E96; F; 0068 0331; # LATIN SMALL LETTER H WITH LINE BELOW
+1E97; F; 0074 0308; # LATIN SMALL LETTER T WITH DIAERESIS
+1E98; F; 0077 030A; # LATIN SMALL LETTER W WITH RING ABOVE
+1E99; F; 0079 030A; # LATIN SMALL LETTER Y WITH RING ABOVE
+1E9A; F; 0061 02BE; # LATIN SMALL LETTER A WITH RIGHT HALF RING
+1E9B; C; 1E61; # LATIN SMALL LETTER LONG S WITH DOT ABOVE
+1E9E; F; 0073 0073; # LATIN CAPITAL LETTER SHARP S
+1E9E; S; 00DF; # LATIN CAPITAL LETTER SHARP S
+1EA0; C; 1EA1; # LATIN CAPITAL LETTER A WITH DOT BELOW
+1EA2; C; 1EA3; # LATIN CAPITAL LETTER A WITH HOOK ABOVE
+1EA4; C; 1EA5; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND ACUTE
+1EA6; C; 1EA7; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND GRAVE
+1EA8; C; 1EA9; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND HOOK ABOVE
+1EAA; C; 1EAB; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND TILDE
+1EAC; C; 1EAD; # LATIN CAPITAL LETTER A WITH CIRCUMFLEX AND DOT BELOW
+1EAE; C; 1EAF; # LATIN CAPITAL LETTER A WITH BREVE AND ACUTE
+1EB0; C; 1EB1; # LATIN CAPITAL LETTER A WITH BREVE AND GRAVE
+1EB2; C; 1EB3; # LATIN CAPITAL LETTER A WITH BREVE AND HOOK ABOVE
+1EB4; C; 1EB5; # LATIN CAPITAL LETTER A WITH BREVE AND TILDE
+1EB6; C; 1EB7; # LATIN CAPITAL LETTER A WITH BREVE AND DOT BELOW
+1EB8; C; 1EB9; # LATIN CAPITAL LETTER E WITH DOT BELOW
+1EBA; C; 1EBB; # LATIN CAPITAL LETTER E WITH HOOK ABOVE
+1EBC; C; 1EBD; # LATIN CAPITAL LETTER E WITH TILDE
+1EBE; C; 1EBF; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND ACUTE
+1EC0; C; 1EC1; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND GRAVE
+1EC2; C; 1EC3; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND HOOK ABOVE
+1EC4; C; 1EC5; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND TILDE
+1EC6; C; 1EC7; # LATIN CAPITAL LETTER E WITH CIRCUMFLEX AND DOT BELOW
+1EC8; C; 1EC9; # LATIN CAPITAL LETTER I WITH HOOK ABOVE
+1ECA; C; 1ECB; # LATIN CAPITAL LETTER I WITH DOT BELOW
+1ECC; C; 1ECD; # LATIN CAPITAL LETTER O WITH DOT BELOW
+1ECE; C; 1ECF; # LATIN CAPITAL LETTER O WITH HOOK ABOVE
+1ED0; C; 1ED1; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND ACUTE
+1ED2; C; 1ED3; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND GRAVE
+1ED4; C; 1ED5; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND HOOK ABOVE
+1ED6; C; 1ED7; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND TILDE
+1ED8; C; 1ED9; # LATIN CAPITAL LETTER O WITH CIRCUMFLEX AND DOT BELOW
+1EDA; C; 1EDB; # LATIN CAPITAL LETTER O WITH HORN AND ACUTE
+1EDC; C; 1EDD; # LATIN CAPITAL LETTER O WITH HORN AND GRAVE
+1EDE; C; 1EDF; # LATIN CAPITAL LETTER O WITH HORN AND HOOK ABOVE
+1EE0; C; 1EE1; # LATIN CAPITAL LETTER O WITH HORN AND TILDE
+1EE2; C; 1EE3; # LATIN CAPITAL LETTER O WITH HORN AND DOT BELOW
+1EE4; C; 1EE5; # LATIN CAPITAL LETTER U WITH DOT BELOW
+1EE6; C; 1EE7; # LATIN CAPITAL LETTER U WITH HOOK ABOVE
+1EE8; C; 1EE9; # LATIN CAPITAL LETTER U WITH HORN AND ACUTE
+1EEA; C; 1EEB; # LATIN CAPITAL LETTER U WITH HORN AND GRAVE
+1EEC; C; 1EED; # LATIN CAPITAL LETTER U WITH HORN AND HOOK ABOVE
+1EEE; C; 1EEF; # LATIN CAPITAL LETTER U WITH HORN AND TILDE
+1EF0; C; 1EF1; # LATIN CAPITAL LETTER U WITH HORN AND DOT BELOW
+1EF2; C; 1EF3; # LATIN CAPITAL LETTER Y WITH GRAVE
+1EF4; C; 1EF5; # LATIN CAPITAL LETTER Y WITH DOT BELOW
+1EF6; C; 1EF7; # LATIN CAPITAL LETTER Y WITH HOOK ABOVE
+1EF8; C; 1EF9; # LATIN CAPITAL LETTER Y WITH TILDE
+1EFA; C; 1EFB; # LATIN CAPITAL LETTER MIDDLE-WELSH LL
+1EFC; C; 1EFD; # LATIN CAPITAL LETTER MIDDLE-WELSH V
+1EFE; C; 1EFF; # LATIN CAPITAL LETTER Y WITH LOOP
+1F08; C; 1F00; # GREEK CAPITAL LETTER ALPHA WITH PSILI
+1F09; C; 1F01; # GREEK CAPITAL LETTER ALPHA WITH DASIA
+1F0A; C; 1F02; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA
+1F0B; C; 1F03; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA
+1F0C; C; 1F04; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA
+1F0D; C; 1F05; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA
+1F0E; C; 1F06; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI
+1F0F; C; 1F07; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI
+1F18; C; 1F10; # GREEK CAPITAL LETTER EPSILON WITH PSILI
+1F19; C; 1F11; # GREEK CAPITAL LETTER EPSILON WITH DASIA
+1F1A; C; 1F12; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND VARIA
+1F1B; C; 1F13; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND VARIA
+1F1C; C; 1F14; # GREEK CAPITAL LETTER EPSILON WITH PSILI AND OXIA
+1F1D; C; 1F15; # GREEK CAPITAL LETTER EPSILON WITH DASIA AND OXIA
+1F28; C; 1F20; # GREEK CAPITAL LETTER ETA WITH PSILI
+1F29; C; 1F21; # GREEK CAPITAL LETTER ETA WITH DASIA
+1F2A; C; 1F22; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA
+1F2B; C; 1F23; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA
+1F2C; C; 1F24; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA
+1F2D; C; 1F25; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA
+1F2E; C; 1F26; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI
+1F2F; C; 1F27; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI
+1F38; C; 1F30; # GREEK CAPITAL LETTER IOTA WITH PSILI
+1F39; C; 1F31; # GREEK CAPITAL LETTER IOTA WITH DASIA
+1F3A; C; 1F32; # GREEK CAPITAL LETTER IOTA WITH PSILI AND VARIA
+1F3B; C; 1F33; # GREEK CAPITAL LETTER IOTA WITH DASIA AND VARIA
+1F3C; C; 1F34; # GREEK CAPITAL LETTER IOTA WITH PSILI AND OXIA
+1F3D; C; 1F35; # GREEK CAPITAL LETTER IOTA WITH DASIA AND OXIA
+1F3E; C; 1F36; # GREEK CAPITAL LETTER IOTA WITH PSILI AND PERISPOMENI
+1F3F; C; 1F37; # GREEK CAPITAL LETTER IOTA WITH DASIA AND PERISPOMENI
+1F48; C; 1F40; # GREEK CAPITAL LETTER OMICRON WITH PSILI
+1F49; C; 1F41; # GREEK CAPITAL LETTER OMICRON WITH DASIA
+1F4A; C; 1F42; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND VARIA
+1F4B; C; 1F43; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND VARIA
+1F4C; C; 1F44; # GREEK CAPITAL LETTER OMICRON WITH PSILI AND OXIA
+1F4D; C; 1F45; # GREEK CAPITAL LETTER OMICRON WITH DASIA AND OXIA
+1F50; F; 03C5 0313; # GREEK SMALL LETTER UPSILON WITH PSILI
+1F52; F; 03C5 0313 0300; # GREEK SMALL LETTER UPSILON WITH PSILI AND VARIA
+1F54; F; 03C5 0313 0301; # GREEK SMALL LETTER UPSILON WITH PSILI AND OXIA
+1F56; F; 03C5 0313 0342; # GREEK SMALL LETTER UPSILON WITH PSILI AND PERISPOMENI
+1F59; C; 1F51; # GREEK CAPITAL LETTER UPSILON WITH DASIA
+1F5B; C; 1F53; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND VARIA
+1F5D; C; 1F55; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND OXIA
+1F5F; C; 1F57; # GREEK CAPITAL LETTER UPSILON WITH DASIA AND PERISPOMENI
+1F68; C; 1F60; # GREEK CAPITAL LETTER OMEGA WITH PSILI
+1F69; C; 1F61; # GREEK CAPITAL LETTER OMEGA WITH DASIA
+1F6A; C; 1F62; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA
+1F6B; C; 1F63; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA
+1F6C; C; 1F64; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA
+1F6D; C; 1F65; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA
+1F6E; C; 1F66; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI
+1F6F; C; 1F67; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI
+1F80; F; 1F00 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND YPOGEGRAMMENI
+1F81; F; 1F01 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND YPOGEGRAMMENI
+1F82; F; 1F02 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F83; F; 1F03 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F84; F; 1F04 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F85; F; 1F05 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F86; F; 1F06 03B9; # GREEK SMALL LETTER ALPHA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F87; F; 1F07 03B9; # GREEK SMALL LETTER ALPHA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F88; F; 1F00 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F88; S; 1F80; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PROSGEGRAMMENI
+1F89; F; 1F01 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F89; S; 1F81; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PROSGEGRAMMENI
+1F8A; F; 1F02 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8A; S; 1F82; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F8B; F; 1F03 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8B; S; 1F83; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F8C; F; 1F04 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8C; S; 1F84; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F8D; F; 1F05 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8D; S; 1F85; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F8E; F; 1F06 03B9; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8E; S; 1F86; # GREEK CAPITAL LETTER ALPHA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; F; 1F07 03B9; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F8F; S; 1F87; # GREEK CAPITAL LETTER ALPHA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F90; F; 1F20 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND YPOGEGRAMMENI
+1F91; F; 1F21 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND YPOGEGRAMMENI
+1F92; F; 1F22 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1F93; F; 1F23 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1F94; F; 1F24 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1F95; F; 1F25 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1F96; F; 1F26 03B9; # GREEK SMALL LETTER ETA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1F97; F; 1F27 03B9; # GREEK SMALL LETTER ETA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1F98; F; 1F20 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F98; S; 1F90; # GREEK CAPITAL LETTER ETA WITH PSILI AND PROSGEGRAMMENI
+1F99; F; 1F21 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F99; S; 1F91; # GREEK CAPITAL LETTER ETA WITH DASIA AND PROSGEGRAMMENI
+1F9A; F; 1F22 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9A; S; 1F92; # GREEK CAPITAL LETTER ETA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1F9B; F; 1F23 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9B; S; 1F93; # GREEK CAPITAL LETTER ETA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1F9C; F; 1F24 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9C; S; 1F94; # GREEK CAPITAL LETTER ETA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1F9D; F; 1F25 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9D; S; 1F95; # GREEK CAPITAL LETTER ETA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1F9E; F; 1F26 03B9; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9E; S; 1F96; # GREEK CAPITAL LETTER ETA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; F; 1F27 03B9; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1F9F; S; 1F97; # GREEK CAPITAL LETTER ETA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FA0; F; 1F60 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND YPOGEGRAMMENI
+1FA1; F; 1F61 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND YPOGEGRAMMENI
+1FA2; F; 1F62 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND VARIA AND YPOGEGRAMMENI
+1FA3; F; 1F63 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND VARIA AND YPOGEGRAMMENI
+1FA4; F; 1F64 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND OXIA AND YPOGEGRAMMENI
+1FA5; F; 1F65 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND OXIA AND YPOGEGRAMMENI
+1FA6; F; 1F66 03B9; # GREEK SMALL LETTER OMEGA WITH PSILI AND PERISPOMENI AND YPOGEGRAMMENI
+1FA7; F; 1F67 03B9; # GREEK SMALL LETTER OMEGA WITH DASIA AND PERISPOMENI AND YPOGEGRAMMENI
+1FA8; F; 1F60 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA8; S; 1FA0; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PROSGEGRAMMENI
+1FA9; F; 1F61 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FA9; S; 1FA1; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PROSGEGRAMMENI
+1FAA; F; 1F62 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAA; S; 1FA2; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND VARIA AND PROSGEGRAMMENI
+1FAB; F; 1F63 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAB; S; 1FA3; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND VARIA AND PROSGEGRAMMENI
+1FAC; F; 1F64 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAC; S; 1FA4; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND OXIA AND PROSGEGRAMMENI
+1FAD; F; 1F65 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAD; S; 1FA5; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND OXIA AND PROSGEGRAMMENI
+1FAE; F; 1F66 03B9; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAE; S; 1FA6; # GREEK CAPITAL LETTER OMEGA WITH PSILI AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; F; 1F67 03B9; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FAF; S; 1FA7; # GREEK CAPITAL LETTER OMEGA WITH DASIA AND PERISPOMENI AND PROSGEGRAMMENI
+1FB2; F; 1F70 03B9; # GREEK SMALL LETTER ALPHA WITH VARIA AND YPOGEGRAMMENI
+1FB3; F; 03B1 03B9; # GREEK SMALL LETTER ALPHA WITH YPOGEGRAMMENI
+1FB4; F; 03AC 03B9; # GREEK SMALL LETTER ALPHA WITH OXIA AND YPOGEGRAMMENI
+1FB6; F; 03B1 0342; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI
+1FB7; F; 03B1 0342 03B9; # GREEK SMALL LETTER ALPHA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FB8; C; 1FB0; # GREEK CAPITAL LETTER ALPHA WITH VRACHY
+1FB9; C; 1FB1; # GREEK CAPITAL LETTER ALPHA WITH MACRON
+1FBA; C; 1F70; # GREEK CAPITAL LETTER ALPHA WITH VARIA
+1FBB; C; 1F71; # GREEK CAPITAL LETTER ALPHA WITH OXIA
+1FBC; F; 03B1 03B9; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBC; S; 1FB3; # GREEK CAPITAL LETTER ALPHA WITH PROSGEGRAMMENI
+1FBE; C; 03B9; # GREEK PROSGEGRAMMENI
+1FC2; F; 1F74 03B9; # GREEK SMALL LETTER ETA WITH VARIA AND YPOGEGRAMMENI
+1FC3; F; 03B7 03B9; # GREEK SMALL LETTER ETA WITH YPOGEGRAMMENI
+1FC4; F; 03AE 03B9; # GREEK SMALL LETTER ETA WITH OXIA AND YPOGEGRAMMENI
+1FC6; F; 03B7 0342; # GREEK SMALL LETTER ETA WITH PERISPOMENI
+1FC7; F; 03B7 0342 03B9; # GREEK SMALL LETTER ETA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FC8; C; 1F72; # GREEK CAPITAL LETTER EPSILON WITH VARIA
+1FC9; C; 1F73; # GREEK CAPITAL LETTER EPSILON WITH OXIA
+1FCA; C; 1F74; # GREEK CAPITAL LETTER ETA WITH VARIA
+1FCB; C; 1F75; # GREEK CAPITAL LETTER ETA WITH OXIA
+1FCC; F; 03B7 03B9; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FCC; S; 1FC3; # GREEK CAPITAL LETTER ETA WITH PROSGEGRAMMENI
+1FD2; F; 03B9 0308 0300; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND VARIA
+1FD3; F; 03B9 0308 0301; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND OXIA
+1FD6; F; 03B9 0342; # GREEK SMALL LETTER IOTA WITH PERISPOMENI
+1FD7; F; 03B9 0308 0342; # GREEK SMALL LETTER IOTA WITH DIALYTIKA AND PERISPOMENI
+1FD8; C; 1FD0; # GREEK CAPITAL LETTER IOTA WITH VRACHY
+1FD9; C; 1FD1; # GREEK CAPITAL LETTER IOTA WITH MACRON
+1FDA; C; 1F76; # GREEK CAPITAL LETTER IOTA WITH VARIA
+1FDB; C; 1F77; # GREEK CAPITAL LETTER IOTA WITH OXIA
+1FE2; F; 03C5 0308 0300; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND VARIA
+1FE3; F; 03C5 0308 0301; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND OXIA
+1FE4; F; 03C1 0313; # GREEK SMALL LETTER RHO WITH PSILI
+1FE6; F; 03C5 0342; # GREEK SMALL LETTER UPSILON WITH PERISPOMENI
+1FE7; F; 03C5 0308 0342; # GREEK SMALL LETTER UPSILON WITH DIALYTIKA AND PERISPOMENI
+1FE8; C; 1FE0; # GREEK CAPITAL LETTER UPSILON WITH VRACHY
+1FE9; C; 1FE1; # GREEK CAPITAL LETTER UPSILON WITH MACRON
+1FEA; C; 1F7A; # GREEK CAPITAL LETTER UPSILON WITH VARIA
+1FEB; C; 1F7B; # GREEK CAPITAL LETTER UPSILON WITH OXIA
+1FEC; C; 1FE5; # GREEK CAPITAL LETTER RHO WITH DASIA
+1FF2; F; 1F7C 03B9; # GREEK SMALL LETTER OMEGA WITH VARIA AND YPOGEGRAMMENI
+1FF3; F; 03C9 03B9; # GREEK SMALL LETTER OMEGA WITH YPOGEGRAMMENI
+1FF4; F; 03CE 03B9; # GREEK SMALL LETTER OMEGA WITH OXIA AND YPOGEGRAMMENI
+1FF6; F; 03C9 0342; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI
+1FF7; F; 03C9 0342 03B9; # GREEK SMALL LETTER OMEGA WITH PERISPOMENI AND YPOGEGRAMMENI
+1FF8; C; 1F78; # GREEK CAPITAL LETTER OMICRON WITH VARIA
+1FF9; C; 1F79; # GREEK CAPITAL LETTER OMICRON WITH OXIA
+1FFA; C; 1F7C; # GREEK CAPITAL LETTER OMEGA WITH VARIA
+1FFB; C; 1F7D; # GREEK CAPITAL LETTER OMEGA WITH OXIA
+1FFC; F; 03C9 03B9; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+1FFC; S; 1FF3; # GREEK CAPITAL LETTER OMEGA WITH PROSGEGRAMMENI
+2126; C; 03C9; # OHM SIGN
+212A; C; 006B; # KELVIN SIGN
+212B; C; 00E5; # ANGSTROM SIGN
+2132; C; 214E; # TURNED CAPITAL F
+2160; C; 2170; # ROMAN NUMERAL ONE
+2161; C; 2171; # ROMAN NUMERAL TWO
+2162; C; 2172; # ROMAN NUMERAL THREE
+2163; C; 2173; # ROMAN NUMERAL FOUR
+2164; C; 2174; # ROMAN NUMERAL FIVE
+2165; C; 2175; # ROMAN NUMERAL SIX
+2166; C; 2176; # ROMAN NUMERAL SEVEN
+2167; C; 2177; # ROMAN NUMERAL EIGHT
+2168; C; 2178; # ROMAN NUMERAL NINE
+2169; C; 2179; # ROMAN NUMERAL TEN
+216A; C; 217A; # ROMAN NUMERAL ELEVEN
+216B; C; 217B; # ROMAN NUMERAL TWELVE
+216C; C; 217C; # ROMAN NUMERAL FIFTY
+216D; C; 217D; # ROMAN NUMERAL ONE HUNDRED
+216E; C; 217E; # ROMAN NUMERAL FIVE HUNDRED
+216F; C; 217F; # ROMAN NUMERAL ONE THOUSAND
+2183; C; 2184; # ROMAN NUMERAL REVERSED ONE HUNDRED
+24B6; C; 24D0; # CIRCLED LATIN CAPITAL LETTER A
+24B7; C; 24D1; # CIRCLED LATIN CAPITAL LETTER B
+24B8; C; 24D2; # CIRCLED LATIN CAPITAL LETTER C
+24B9; C; 24D3; # CIRCLED LATIN CAPITAL LETTER D
+24BA; C; 24D4; # CIRCLED LATIN CAPITAL LETTER E
+24BB; C; 24D5; # CIRCLED LATIN CAPITAL LETTER F
+24BC; C; 24D6; # CIRCLED LATIN CAPITAL LETTER G
+24BD; C; 24D7; # CIRCLED LATIN CAPITAL LETTER H
+24BE; C; 24D8; # CIRCLED LATIN CAPITAL LETTER I
+24BF; C; 24D9; # CIRCLED LATIN CAPITAL LETTER J
+24C0; C; 24DA; # CIRCLED LATIN CAPITAL LETTER K
+24C1; C; 24DB; # CIRCLED LATIN CAPITAL LETTER L
+24C2; C; 24DC; # CIRCLED LATIN CAPITAL LETTER M
+24C3; C; 24DD; # CIRCLED LATIN CAPITAL LETTER N
+24C4; C; 24DE; # CIRCLED LATIN CAPITAL LETTER O
+24C5; C; 24DF; # CIRCLED LATIN CAPITAL LETTER P
+24C6; C; 24E0; # CIRCLED LATIN CAPITAL LETTER Q
+24C7; C; 24E1; # CIRCLED LATIN CAPITAL LETTER R
+24C8; C; 24E2; # CIRCLED LATIN CAPITAL LETTER S
+24C9; C; 24E3; # CIRCLED LATIN CAPITAL LETTER T
+24CA; C; 24E4; # CIRCLED LATIN CAPITAL LETTER U
+24CB; C; 24E5; # CIRCLED LATIN CAPITAL LETTER V
+24CC; C; 24E6; # CIRCLED LATIN CAPITAL LETTER W
+24CD; C; 24E7; # CIRCLED LATIN CAPITAL LETTER X
+24CE; C; 24E8; # CIRCLED LATIN CAPITAL LETTER Y
+24CF; C; 24E9; # CIRCLED LATIN CAPITAL LETTER Z
+2C00; C; 2C30; # GLAGOLITIC CAPITAL LETTER AZU
+2C01; C; 2C31; # GLAGOLITIC CAPITAL LETTER BUKY
+2C02; C; 2C32; # GLAGOLITIC CAPITAL LETTER VEDE
+2C03; C; 2C33; # GLAGOLITIC CAPITAL LETTER GLAGOLI
+2C04; C; 2C34; # GLAGOLITIC CAPITAL LETTER DOBRO
+2C05; C; 2C35; # GLAGOLITIC CAPITAL LETTER YESTU
+2C06; C; 2C36; # GLAGOLITIC CAPITAL LETTER ZHIVETE
+2C07; C; 2C37; # GLAGOLITIC CAPITAL LETTER DZELO
+2C08; C; 2C38; # GLAGOLITIC CAPITAL LETTER ZEMLJA
+2C09; C; 2C39; # GLAGOLITIC CAPITAL LETTER IZHE
+2C0A; C; 2C3A; # GLAGOLITIC CAPITAL LETTER INITIAL IZHE
+2C0B; C; 2C3B; # GLAGOLITIC CAPITAL LETTER I
+2C0C; C; 2C3C; # GLAGOLITIC CAPITAL LETTER DJERVI
+2C0D; C; 2C3D; # GLAGOLITIC CAPITAL LETTER KAKO
+2C0E; C; 2C3E; # GLAGOLITIC CAPITAL LETTER LJUDIJE
+2C0F; C; 2C3F; # GLAGOLITIC CAPITAL LETTER MYSLITE
+2C10; C; 2C40; # GLAGOLITIC CAPITAL LETTER NASHI
+2C11; C; 2C41; # GLAGOLITIC CAPITAL LETTER ONU
+2C12; C; 2C42; # GLAGOLITIC CAPITAL LETTER POKOJI
+2C13; C; 2C43; # GLAGOLITIC CAPITAL LETTER RITSI
+2C14; C; 2C44; # GLAGOLITIC CAPITAL LETTER SLOVO
+2C15; C; 2C45; # GLAGOLITIC CAPITAL LETTER TVRIDO
+2C16; C; 2C46; # GLAGOLITIC CAPITAL LETTER UKU
+2C17; C; 2C47; # GLAGOLITIC CAPITAL LETTER FRITU
+2C18; C; 2C48; # GLAGOLITIC CAPITAL LETTER HERU
+2C19; C; 2C49; # GLAGOLITIC CAPITAL LETTER OTU
+2C1A; C; 2C4A; # GLAGOLITIC CAPITAL LETTER PE
+2C1B; C; 2C4B; # GLAGOLITIC CAPITAL LETTER SHTA
+2C1C; C; 2C4C; # GLAGOLITIC CAPITAL LETTER TSI
+2C1D; C; 2C4D; # GLAGOLITIC CAPITAL LETTER CHRIVI
+2C1E; C; 2C4E; # GLAGOLITIC CAPITAL LETTER SHA
+2C1F; C; 2C4F; # GLAGOLITIC CAPITAL LETTER YERU
+2C20; C; 2C50; # GLAGOLITIC CAPITAL LETTER YERI
+2C21; C; 2C51; # GLAGOLITIC CAPITAL LETTER YATI
+2C22; C; 2C52; # GLAGOLITIC CAPITAL LETTER SPIDERY HA
+2C23; C; 2C53; # GLAGOLITIC CAPITAL LETTER YU
+2C24; C; 2C54; # GLAGOLITIC CAPITAL LETTER SMALL YUS
+2C25; C; 2C55; # GLAGOLITIC CAPITAL LETTER SMALL YUS WITH TAIL
+2C26; C; 2C56; # GLAGOLITIC CAPITAL LETTER YO
+2C27; C; 2C57; # GLAGOLITIC CAPITAL LETTER IOTATED SMALL YUS
+2C28; C; 2C58; # GLAGOLITIC CAPITAL LETTER BIG YUS
+2C29; C; 2C59; # GLAGOLITIC CAPITAL LETTER IOTATED BIG YUS
+2C2A; C; 2C5A; # GLAGOLITIC CAPITAL LETTER FITA
+2C2B; C; 2C5B; # GLAGOLITIC CAPITAL LETTER IZHITSA
+2C2C; C; 2C5C; # GLAGOLITIC CAPITAL LETTER SHTAPIC
+2C2D; C; 2C5D; # GLAGOLITIC CAPITAL LETTER TROKUTASTI A
+2C2E; C; 2C5E; # GLAGOLITIC CAPITAL LETTER LATINATE MYSLITE
+2C2F; C; 2C5F; # GLAGOLITIC CAPITAL LETTER CAUDATE CHRIVI
+2C60; C; 2C61; # LATIN CAPITAL LETTER L WITH DOUBLE BAR
+2C62; C; 026B; # LATIN CAPITAL LETTER L WITH MIDDLE TILDE
+2C63; C; 1D7D; # LATIN CAPITAL LETTER P WITH STROKE
+2C64; C; 027D; # LATIN CAPITAL LETTER R WITH TAIL
+2C67; C; 2C68; # LATIN CAPITAL LETTER H WITH DESCENDER
+2C69; C; 2C6A; # LATIN CAPITAL LETTER K WITH DESCENDER
+2C6B; C; 2C6C; # LATIN CAPITAL LETTER Z WITH DESCENDER
+2C6D; C; 0251; # LATIN CAPITAL LETTER ALPHA
+2C6E; C; 0271; # LATIN CAPITAL LETTER M WITH HOOK
+2C6F; C; 0250; # LATIN CAPITAL LETTER TURNED A
+2C70; C; 0252; # LATIN CAPITAL LETTER TURNED ALPHA
+2C72; C; 2C73; # LATIN CAPITAL LETTER W WITH HOOK
+2C75; C; 2C76; # LATIN CAPITAL LETTER HALF H
+2C7E; C; 023F; # LATIN CAPITAL LETTER S WITH SWASH TAIL
+2C7F; C; 0240; # LATIN CAPITAL LETTER Z WITH SWASH TAIL
+2C80; C; 2C81; # COPTIC CAPITAL LETTER ALFA
+2C82; C; 2C83; # COPTIC CAPITAL LETTER VIDA
+2C84; C; 2C85; # COPTIC CAPITAL LETTER GAMMA
+2C86; C; 2C87; # COPTIC CAPITAL LETTER DALDA
+2C88; C; 2C89; # COPTIC CAPITAL LETTER EIE
+2C8A; C; 2C8B; # COPTIC CAPITAL LETTER SOU
+2C8C; C; 2C8D; # COPTIC CAPITAL LETTER ZATA
+2C8E; C; 2C8F; # COPTIC CAPITAL LETTER HATE
+2C90; C; 2C91; # COPTIC CAPITAL LETTER THETHE
+2C92; C; 2C93; # COPTIC CAPITAL LETTER IAUDA
+2C94; C; 2C95; # COPTIC CAPITAL LETTER KAPA
+2C96; C; 2C97; # COPTIC CAPITAL LETTER LAULA
+2C98; C; 2C99; # COPTIC CAPITAL LETTER MI
+2C9A; C; 2C9B; # COPTIC CAPITAL LETTER NI
+2C9C; C; 2C9D; # COPTIC CAPITAL LETTER KSI
+2C9E; C; 2C9F; # COPTIC CAPITAL LETTER O
+2CA0; C; 2CA1; # COPTIC CAPITAL LETTER PI
+2CA2; C; 2CA3; # COPTIC CAPITAL LETTER RO
+2CA4; C; 2CA5; # COPTIC CAPITAL LETTER SIMA
+2CA6; C; 2CA7; # COPTIC CAPITAL LETTER TAU
+2CA8; C; 2CA9; # COPTIC CAPITAL LETTER UA
+2CAA; C; 2CAB; # COPTIC CAPITAL LETTER FI
+2CAC; C; 2CAD; # COPTIC CAPITAL LETTER KHI
+2CAE; C; 2CAF; # COPTIC CAPITAL LETTER PSI
+2CB0; C; 2CB1; # COPTIC CAPITAL LETTER OOU
+2CB2; C; 2CB3; # COPTIC CAPITAL LETTER DIALECT-P ALEF
+2CB4; C; 2CB5; # COPTIC CAPITAL LETTER OLD COPTIC AIN
+2CB6; C; 2CB7; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC EIE
+2CB8; C; 2CB9; # COPTIC CAPITAL LETTER DIALECT-P KAPA
+2CBA; C; 2CBB; # COPTIC CAPITAL LETTER DIALECT-P NI
+2CBC; C; 2CBD; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC NI
+2CBE; C; 2CBF; # COPTIC CAPITAL LETTER OLD COPTIC OOU
+2CC0; C; 2CC1; # COPTIC CAPITAL LETTER SAMPI
+2CC2; C; 2CC3; # COPTIC CAPITAL LETTER CROSSED SHEI
+2CC4; C; 2CC5; # COPTIC CAPITAL LETTER OLD COPTIC SHEI
+2CC6; C; 2CC7; # COPTIC CAPITAL LETTER OLD COPTIC ESH
+2CC8; C; 2CC9; # COPTIC CAPITAL LETTER AKHMIMIC KHEI
+2CCA; C; 2CCB; # COPTIC CAPITAL LETTER DIALECT-P HORI
+2CCC; C; 2CCD; # COPTIC CAPITAL LETTER OLD COPTIC HORI
+2CCE; C; 2CCF; # COPTIC CAPITAL LETTER OLD COPTIC HA
+2CD0; C; 2CD1; # COPTIC CAPITAL LETTER L-SHAPED HA
+2CD2; C; 2CD3; # COPTIC CAPITAL LETTER OLD COPTIC HEI
+2CD4; C; 2CD5; # COPTIC CAPITAL LETTER OLD COPTIC HAT
+2CD6; C; 2CD7; # COPTIC CAPITAL LETTER OLD COPTIC GANGIA
+2CD8; C; 2CD9; # COPTIC CAPITAL LETTER OLD COPTIC DJA
+2CDA; C; 2CDB; # COPTIC CAPITAL LETTER OLD COPTIC SHIMA
+2CDC; C; 2CDD; # COPTIC CAPITAL LETTER OLD NUBIAN SHIMA
+2CDE; C; 2CDF; # COPTIC CAPITAL LETTER OLD NUBIAN NGI
+2CE0; C; 2CE1; # COPTIC CAPITAL LETTER OLD NUBIAN NYI
+2CE2; C; 2CE3; # COPTIC CAPITAL LETTER OLD NUBIAN WAU
+2CEB; C; 2CEC; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC SHEI
+2CED; C; 2CEE; # COPTIC CAPITAL LETTER CRYPTOGRAMMIC GANGIA
+2CF2; C; 2CF3; # COPTIC CAPITAL LETTER BOHAIRIC KHEI
+A640; C; A641; # CYRILLIC CAPITAL LETTER ZEMLYA
+A642; C; A643; # CYRILLIC CAPITAL LETTER DZELO
+A644; C; A645; # CYRILLIC CAPITAL LETTER REVERSED DZE
+A646; C; A647; # CYRILLIC CAPITAL LETTER IOTA
+A648; C; A649; # CYRILLIC CAPITAL LETTER DJERV
+A64A; C; A64B; # CYRILLIC CAPITAL LETTER MONOGRAPH UK
+A64C; C; A64D; # CYRILLIC CAPITAL LETTER BROAD OMEGA
+A64E; C; A64F; # CYRILLIC CAPITAL LETTER NEUTRAL YER
+A650; C; A651; # CYRILLIC CAPITAL LETTER YERU WITH BACK YER
+A652; C; A653; # CYRILLIC CAPITAL LETTER IOTIFIED YAT
+A654; C; A655; # CYRILLIC CAPITAL LETTER REVERSED YU
+A656; C; A657; # CYRILLIC CAPITAL LETTER IOTIFIED A
+A658; C; A659; # CYRILLIC CAPITAL LETTER CLOSED LITTLE YUS
+A65A; C; A65B; # CYRILLIC CAPITAL LETTER BLENDED YUS
+A65C; C; A65D; # CYRILLIC CAPITAL LETTER IOTIFIED CLOSED LITTLE YUS
+A65E; C; A65F; # CYRILLIC CAPITAL LETTER YN
+A660; C; A661; # CYRILLIC CAPITAL LETTER REVERSED TSE
+A662; C; A663; # CYRILLIC CAPITAL LETTER SOFT DE
+A664; C; A665; # CYRILLIC CAPITAL LETTER SOFT EL
+A666; C; A667; # CYRILLIC CAPITAL LETTER SOFT EM
+A668; C; A669; # CYRILLIC CAPITAL LETTER MONOCULAR O
+A66A; C; A66B; # CYRILLIC CAPITAL LETTER BINOCULAR O
+A66C; C; A66D; # CYRILLIC CAPITAL LETTER DOUBLE MONOCULAR O
+A680; C; A681; # CYRILLIC CAPITAL LETTER DWE
+A682; C; A683; # CYRILLIC CAPITAL LETTER DZWE
+A684; C; A685; # CYRILLIC CAPITAL LETTER ZHWE
+A686; C; A687; # CYRILLIC CAPITAL LETTER CCHE
+A688; C; A689; # CYRILLIC CAPITAL LETTER DZZE
+A68A; C; A68B; # CYRILLIC CAPITAL LETTER TE WITH MIDDLE HOOK
+A68C; C; A68D; # CYRILLIC CAPITAL LETTER TWE
+A68E; C; A68F; # CYRILLIC CAPITAL LETTER TSWE
+A690; C; A691; # CYRILLIC CAPITAL LETTER TSSE
+A692; C; A693; # CYRILLIC CAPITAL LETTER TCHE
+A694; C; A695; # CYRILLIC CAPITAL LETTER HWE
+A696; C; A697; # CYRILLIC CAPITAL LETTER SHWE
+A698; C; A699; # CYRILLIC CAPITAL LETTER DOUBLE O
+A69A; C; A69B; # CYRILLIC CAPITAL LETTER CROSSED O
+A722; C; A723; # LATIN CAPITAL LETTER EGYPTOLOGICAL ALEF
+A724; C; A725; # LATIN CAPITAL LETTER EGYPTOLOGICAL AIN
+A726; C; A727; # LATIN CAPITAL LETTER HENG
+A728; C; A729; # LATIN CAPITAL LETTER TZ
+A72A; C; A72B; # LATIN CAPITAL LETTER TRESILLO
+A72C; C; A72D; # LATIN CAPITAL LETTER CUATRILLO
+A72E; C; A72F; # LATIN CAPITAL LETTER CUATRILLO WITH COMMA
+A732; C; A733; # LATIN CAPITAL LETTER AA
+A734; C; A735; # LATIN CAPITAL LETTER AO
+A736; C; A737; # LATIN CAPITAL LETTER AU
+A738; C; A739; # LATIN CAPITAL LETTER AV
+A73A; C; A73B; # LATIN CAPITAL LETTER AV WITH HORIZONTAL BAR
+A73C; C; A73D; # LATIN CAPITAL LETTER AY
+A73E; C; A73F; # LATIN CAPITAL LETTER REVERSED C WITH DOT
+A740; C; A741; # LATIN CAPITAL LETTER K WITH STROKE
+A742; C; A743; # LATIN CAPITAL LETTER K WITH DIAGONAL STROKE
+A744; C; A745; # LATIN CAPITAL LETTER K WITH STROKE AND DIAGONAL STROKE
+A746; C; A747; # LATIN CAPITAL LETTER BROKEN L
+A748; C; A749; # LATIN CAPITAL LETTER L WITH HIGH STROKE
+A74A; C; A74B; # LATIN CAPITAL LETTER O WITH LONG STROKE OVERLAY
+A74C; C; A74D; # LATIN CAPITAL LETTER O WITH LOOP
+A74E; C; A74F; # LATIN CAPITAL LETTER OO
+A750; C; A751; # LATIN CAPITAL LETTER P WITH STROKE THROUGH DESCENDER
+A752; C; A753; # LATIN CAPITAL LETTER P WITH FLOURISH
+A754; C; A755; # LATIN CAPITAL LETTER P WITH SQUIRREL TAIL
+A756; C; A757; # LATIN CAPITAL LETTER Q WITH STROKE THROUGH DESCENDER
+A758; C; A759; # LATIN CAPITAL LETTER Q WITH DIAGONAL STROKE
+A75A; C; A75B; # LATIN CAPITAL LETTER R ROTUNDA
+A75C; C; A75D; # LATIN CAPITAL LETTER RUM ROTUNDA
+A75E; C; A75F; # LATIN CAPITAL LETTER V WITH DIAGONAL STROKE
+A760; C; A761; # LATIN CAPITAL LETTER VY
+A762; C; A763; # LATIN CAPITAL LETTER VISIGOTHIC Z
+A764; C; A765; # LATIN CAPITAL LETTER THORN WITH STROKE
+A766; C; A767; # LATIN CAPITAL LETTER THORN WITH STROKE THROUGH DESCENDER
+A768; C; A769; # LATIN CAPITAL LETTER VEND
+A76A; C; A76B; # LATIN CAPITAL LETTER ET
+A76C; C; A76D; # LATIN CAPITAL LETTER IS
+A76E; C; A76F; # LATIN CAPITAL LETTER CON
+A779; C; A77A; # LATIN CAPITAL LETTER INSULAR D
+A77B; C; A77C; # LATIN CAPITAL LETTER INSULAR F
+A77D; C; 1D79; # LATIN CAPITAL LETTER INSULAR G
+A77E; C; A77F; # LATIN CAPITAL LETTER TURNED INSULAR G
+A780; C; A781; # LATIN CAPITAL LETTER TURNED L
+A782; C; A783; # LATIN CAPITAL LETTER INSULAR R
+A784; C; A785; # LATIN CAPITAL LETTER INSULAR S
+A786; C; A787; # LATIN CAPITAL LETTER INSULAR T
+A78B; C; A78C; # LATIN CAPITAL LETTER SALTILLO
+A78D; C; 0265; # LATIN CAPITAL LETTER TURNED H
+A790; C; A791; # LATIN CAPITAL LETTER N WITH DESCENDER
+A792; C; A793; # LATIN CAPITAL LETTER C WITH BAR
+A796; C; A797; # LATIN CAPITAL LETTER B WITH FLOURISH
+A798; C; A799; # LATIN CAPITAL LETTER F WITH STROKE
+A79A; C; A79B; # LATIN CAPITAL LETTER VOLAPUK AE
+A79C; C; A79D; # LATIN CAPITAL LETTER VOLAPUK OE
+A79E; C; A79F; # LATIN CAPITAL LETTER VOLAPUK UE
+A7A0; C; A7A1; # LATIN CAPITAL LETTER G WITH OBLIQUE STROKE
+A7A2; C; A7A3; # LATIN CAPITAL LETTER K WITH OBLIQUE STROKE
+A7A4; C; A7A5; # LATIN CAPITAL LETTER N WITH OBLIQUE STROKE
+A7A6; C; A7A7; # LATIN CAPITAL LETTER R WITH OBLIQUE STROKE
+A7A8; C; A7A9; # LATIN CAPITAL LETTER S WITH OBLIQUE STROKE
+A7AA; C; 0266; # LATIN CAPITAL LETTER H WITH HOOK
+A7AB; C; 025C; # LATIN CAPITAL LETTER REVERSED OPEN E
+A7AC; C; 0261; # LATIN CAPITAL LETTER SCRIPT G
+A7AD; C; 026C; # LATIN CAPITAL LETTER L WITH BELT
+A7AE; C; 026A; # LATIN CAPITAL LETTER SMALL CAPITAL I
+A7B0; C; 029E; # LATIN CAPITAL LETTER TURNED K
+A7B1; C; 0287; # LATIN CAPITAL LETTER TURNED T
+A7B2; C; 029D; # LATIN CAPITAL LETTER J WITH CROSSED-TAIL
+A7B3; C; AB53; # LATIN CAPITAL LETTER CHI
+A7B4; C; A7B5; # LATIN CAPITAL LETTER BETA
+A7B6; C; A7B7; # LATIN CAPITAL LETTER OMEGA
+A7B8; C; A7B9; # LATIN CAPITAL LETTER U WITH STROKE
+A7BA; C; A7BB; # LATIN CAPITAL LETTER GLOTTAL A
+A7BC; C; A7BD; # LATIN CAPITAL LETTER GLOTTAL I
+A7BE; C; A7BF; # LATIN CAPITAL LETTER GLOTTAL U
+A7C0; C; A7C1; # LATIN CAPITAL LETTER OLD POLISH O
+A7C2; C; A7C3; # LATIN CAPITAL LETTER ANGLICANA W
+A7C4; C; A794; # LATIN CAPITAL LETTER C WITH PALATAL HOOK
+A7C5; C; 0282; # LATIN CAPITAL LETTER S WITH HOOK
+A7C6; C; 1D8E; # LATIN CAPITAL LETTER Z WITH PALATAL HOOK
+A7C7; C; A7C8; # LATIN CAPITAL LETTER D WITH SHORT STROKE OVERLAY
+A7C9; C; A7CA; # LATIN CAPITAL LETTER S WITH SHORT STROKE OVERLAY
+A7D0; C; A7D1; # LATIN CAPITAL LETTER CLOSED INSULAR G
+A7D6; C; A7D7; # LATIN CAPITAL LETTER MIDDLE SCOTS S
+A7D8; C; A7D9; # LATIN CAPITAL LETTER SIGMOID S
+A7F5; C; A7F6; # LATIN CAPITAL LETTER REVERSED HALF H
+AB70; C; 13A0; # CHEROKEE SMALL LETTER A
+AB71; C; 13A1; # CHEROKEE SMALL LETTER E
+AB72; C; 13A2; # CHEROKEE SMALL LETTER I
+AB73; C; 13A3; # CHEROKEE SMALL LETTER O
+AB74; C; 13A4; # CHEROKEE SMALL LETTER U
+AB75; C; 13A5; # CHEROKEE SMALL LETTER V
+AB76; C; 13A6; # CHEROKEE SMALL LETTER GA
+AB77; C; 13A7; # CHEROKEE SMALL LETTER KA
+AB78; C; 13A8; # CHEROKEE SMALL LETTER GE
+AB79; C; 13A9; # CHEROKEE SMALL LETTER GI
+AB7A; C; 13AA; # CHEROKEE SMALL LETTER GO
+AB7B; C; 13AB; # CHEROKEE SMALL LETTER GU
+AB7C; C; 13AC; # CHEROKEE SMALL LETTER GV
+AB7D; C; 13AD; # CHEROKEE SMALL LETTER HA
+AB7E; C; 13AE; # CHEROKEE SMALL LETTER HE
+AB7F; C; 13AF; # CHEROKEE SMALL LETTER HI
+AB80; C; 13B0; # CHEROKEE SMALL LETTER HO
+AB81; C; 13B1; # CHEROKEE SMALL LETTER HU
+AB82; C; 13B2; # CHEROKEE SMALL LETTER HV
+AB83; C; 13B3; # CHEROKEE SMALL LETTER LA
+AB84; C; 13B4; # CHEROKEE SMALL LETTER LE
+AB85; C; 13B5; # CHEROKEE SMALL LETTER LI
+AB86; C; 13B6; # CHEROKEE SMALL LETTER LO
+AB87; C; 13B7; # CHEROKEE SMALL LETTER LU
+AB88; C; 13B8; # CHEROKEE SMALL LETTER LV
+AB89; C; 13B9; # CHEROKEE SMALL LETTER MA
+AB8A; C; 13BA; # CHEROKEE SMALL LETTER ME
+AB8B; C; 13BB; # CHEROKEE SMALL LETTER MI
+AB8C; C; 13BC; # CHEROKEE SMALL LETTER MO
+AB8D; C; 13BD; # CHEROKEE SMALL LETTER MU
+AB8E; C; 13BE; # CHEROKEE SMALL LETTER NA
+AB8F; C; 13BF; # CHEROKEE SMALL LETTER HNA
+AB90; C; 13C0; # CHEROKEE SMALL LETTER NAH
+AB91; C; 13C1; # CHEROKEE SMALL LETTER NE
+AB92; C; 13C2; # CHEROKEE SMALL LETTER NI
+AB93; C; 13C3; # CHEROKEE SMALL LETTER NO
+AB94; C; 13C4; # CHEROKEE SMALL LETTER NU
+AB95; C; 13C5; # CHEROKEE SMALL LETTER NV
+AB96; C; 13C6; # CHEROKEE SMALL LETTER QUA
+AB97; C; 13C7; # CHEROKEE SMALL LETTER QUE
+AB98; C; 13C8; # CHEROKEE SMALL LETTER QUI
+AB99; C; 13C9; # CHEROKEE SMALL LETTER QUO
+AB9A; C; 13CA; # CHEROKEE SMALL LETTER QUU
+AB9B; C; 13CB; # CHEROKEE SMALL LETTER QUV
+AB9C; C; 13CC; # CHEROKEE SMALL LETTER SA
+AB9D; C; 13CD; # CHEROKEE SMALL LETTER S
+AB9E; C; 13CE; # CHEROKEE SMALL LETTER SE
+AB9F; C; 13CF; # CHEROKEE SMALL LETTER SI
+ABA0; C; 13D0; # CHEROKEE SMALL LETTER SO
+ABA1; C; 13D1; # CHEROKEE SMALL LETTER SU
+ABA2; C; 13D2; # CHEROKEE SMALL LETTER SV
+ABA3; C; 13D3; # CHEROKEE SMALL LETTER DA
+ABA4; C; 13D4; # CHEROKEE SMALL LETTER TA
+ABA5; C; 13D5; # CHEROKEE SMALL LETTER DE
+ABA6; C; 13D6; # CHEROKEE SMALL LETTER TE
+ABA7; C; 13D7; # CHEROKEE SMALL LETTER DI
+ABA8; C; 13D8; # CHEROKEE SMALL LETTER TI
+ABA9; C; 13D9; # CHEROKEE SMALL LETTER DO
+ABAA; C; 13DA; # CHEROKEE SMALL LETTER DU
+ABAB; C; 13DB; # CHEROKEE SMALL LETTER DV
+ABAC; C; 13DC; # CHEROKEE SMALL LETTER DLA
+ABAD; C; 13DD; # CHEROKEE SMALL LETTER TLA
+ABAE; C; 13DE; # CHEROKEE SMALL LETTER TLE
+ABAF; C; 13DF; # CHEROKEE SMALL LETTER TLI
+ABB0; C; 13E0; # CHEROKEE SMALL LETTER TLO
+ABB1; C; 13E1; # CHEROKEE SMALL LETTER TLU
+ABB2; C; 13E2; # CHEROKEE SMALL LETTER TLV
+ABB3; C; 13E3; # CHEROKEE SMALL LETTER TSA
+ABB4; C; 13E4; # CHEROKEE SMALL LETTER TSE
+ABB5; C; 13E5; # CHEROKEE SMALL LETTER TSI
+ABB6; C; 13E6; # CHEROKEE SMALL LETTER TSO
+ABB7; C; 13E7; # CHEROKEE SMALL LETTER TSU
+ABB8; C; 13E8; # CHEROKEE SMALL LETTER TSV
+ABB9; C; 13E9; # CHEROKEE SMALL LETTER WA
+ABBA; C; 13EA; # CHEROKEE SMALL LETTER WE
+ABBB; C; 13EB; # CHEROKEE SMALL LETTER WI
+ABBC; C; 13EC; # CHEROKEE SMALL LETTER WO
+ABBD; C; 13ED; # CHEROKEE SMALL LETTER WU
+ABBE; C; 13EE; # CHEROKEE SMALL LETTER WV
+ABBF; C; 13EF; # CHEROKEE SMALL LETTER YA
+FB00; F; 0066 0066; # LATIN SMALL LIGATURE FF
+FB01; F; 0066 0069; # LATIN SMALL LIGATURE FI
+FB02; F; 0066 006C; # LATIN SMALL LIGATURE FL
+FB03; F; 0066 0066 0069; # LATIN SMALL LIGATURE FFI
+FB04; F; 0066 0066 006C; # LATIN SMALL LIGATURE FFL
+FB05; F; 0073 0074; # LATIN SMALL LIGATURE LONG S T
+FB06; F; 0073 0074; # LATIN SMALL LIGATURE ST
+FB13; F; 0574 0576; # ARMENIAN SMALL LIGATURE MEN NOW
+FB14; F; 0574 0565; # ARMENIAN SMALL LIGATURE MEN ECH
+FB15; F; 0574 056B; # ARMENIAN SMALL LIGATURE MEN INI
+FB16; F; 057E 0576; # ARMENIAN SMALL LIGATURE VEW NOW
+FB17; F; 0574 056D; # ARMENIAN SMALL LIGATURE MEN XEH
+FF21; C; FF41; # FULLWIDTH LATIN CAPITAL LETTER A
+FF22; C; FF42; # FULLWIDTH LATIN CAPITAL LETTER B
+FF23; C; FF43; # FULLWIDTH LATIN CAPITAL LETTER C
+FF24; C; FF44; # FULLWIDTH LATIN CAPITAL LETTER D
+FF25; C; FF45; # FULLWIDTH LATIN CAPITAL LETTER E
+FF26; C; FF46; # FULLWIDTH LATIN CAPITAL LETTER F
+FF27; C; FF47; # FULLWIDTH LATIN CAPITAL LETTER G
+FF28; C; FF48; # FULLWIDTH LATIN CAPITAL LETTER H
+FF29; C; FF49; # FULLWIDTH LATIN CAPITAL LETTER I
+FF2A; C; FF4A; # FULLWIDTH LATIN CAPITAL LETTER J
+FF2B; C; FF4B; # FULLWIDTH LATIN CAPITAL LETTER K
+FF2C; C; FF4C; # FULLWIDTH LATIN CAPITAL LETTER L
+FF2D; C; FF4D; # FULLWIDTH LATIN CAPITAL LETTER M
+FF2E; C; FF4E; # FULLWIDTH LATIN CAPITAL LETTER N
+FF2F; C; FF4F; # FULLWIDTH LATIN CAPITAL LETTER O
+FF30; C; FF50; # FULLWIDTH LATIN CAPITAL LETTER P
+FF31; C; FF51; # FULLWIDTH LATIN CAPITAL LETTER Q
+FF32; C; FF52; # FULLWIDTH LATIN CAPITAL LETTER R
+FF33; C; FF53; # FULLWIDTH LATIN CAPITAL LETTER S
+FF34; C; FF54; # FULLWIDTH LATIN CAPITAL LETTER T
+FF35; C; FF55; # FULLWIDTH LATIN CAPITAL LETTER U
+FF36; C; FF56; # FULLWIDTH LATIN CAPITAL LETTER V
+FF37; C; FF57; # FULLWIDTH LATIN CAPITAL LETTER W
+FF38; C; FF58; # FULLWIDTH LATIN CAPITAL LETTER X
+FF39; C; FF59; # FULLWIDTH LATIN CAPITAL LETTER Y
+FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z
+10400; C; 10428; # DESERET CAPITAL LETTER LONG I
+10401; C; 10429; # DESERET CAPITAL LETTER LONG E
+10402; C; 1042A; # DESERET CAPITAL LETTER LONG A
+10403; C; 1042B; # DESERET CAPITAL LETTER LONG AH
+10404; C; 1042C; # DESERET CAPITAL LETTER LONG O
+10405; C; 1042D; # DESERET CAPITAL LETTER LONG OO
+10406; C; 1042E; # DESERET CAPITAL LETTER SHORT I
+10407; C; 1042F; # DESERET CAPITAL LETTER SHORT E
+10408; C; 10430; # DESERET CAPITAL LETTER SHORT A
+10409; C; 10431; # DESERET CAPITAL LETTER SHORT AH
+1040A; C; 10432; # DESERET CAPITAL LETTER SHORT O
+1040B; C; 10433; # DESERET CAPITAL LETTER SHORT OO
+1040C; C; 10434; # DESERET CAPITAL LETTER AY
+1040D; C; 10435; # DESERET CAPITAL LETTER OW
+1040E; C; 10436; # DESERET CAPITAL LETTER WU
+1040F; C; 10437; # DESERET CAPITAL LETTER YEE
+10410; C; 10438; # DESERET CAPITAL LETTER H
+10411; C; 10439; # DESERET CAPITAL LETTER PEE
+10412; C; 1043A; # DESERET CAPITAL LETTER BEE
+10413; C; 1043B; # DESERET CAPITAL LETTER TEE
+10414; C; 1043C; # DESERET CAPITAL LETTER DEE
+10415; C; 1043D; # DESERET CAPITAL LETTER CHEE
+10416; C; 1043E; # DESERET CAPITAL LETTER JEE
+10417; C; 1043F; # DESERET CAPITAL LETTER KAY
+10418; C; 10440; # DESERET CAPITAL LETTER GAY
+10419; C; 10441; # DESERET CAPITAL LETTER EF
+1041A; C; 10442; # DESERET CAPITAL LETTER VEE
+1041B; C; 10443; # DESERET CAPITAL LETTER ETH
+1041C; C; 10444; # DESERET CAPITAL LETTER THEE
+1041D; C; 10445; # DESERET CAPITAL LETTER ES
+1041E; C; 10446; # DESERET CAPITAL LETTER ZEE
+1041F; C; 10447; # DESERET CAPITAL LETTER ESH
+10420; C; 10448; # DESERET CAPITAL LETTER ZHEE
+10421; C; 10449; # DESERET CAPITAL LETTER ER
+10422; C; 1044A; # DESERET CAPITAL LETTER EL
+10423; C; 1044B; # DESERET CAPITAL LETTER EM
+10424; C; 1044C; # DESERET CAPITAL LETTER EN
+10425; C; 1044D; # DESERET CAPITAL LETTER ENG
+10426; C; 1044E; # DESERET CAPITAL LETTER OI
+10427; C; 1044F; # DESERET CAPITAL LETTER EW
+104B0; C; 104D8; # OSAGE CAPITAL LETTER A
+104B1; C; 104D9; # OSAGE CAPITAL LETTER AI
+104B2; C; 104DA; # OSAGE CAPITAL LETTER AIN
+104B3; C; 104DB; # OSAGE CAPITAL LETTER AH
+104B4; C; 104DC; # OSAGE CAPITAL LETTER BRA
+104B5; C; 104DD; # OSAGE CAPITAL LETTER CHA
+104B6; C; 104DE; # OSAGE CAPITAL LETTER EHCHA
+104B7; C; 104DF; # OSAGE CAPITAL LETTER E
+104B8; C; 104E0; # OSAGE CAPITAL LETTER EIN
+104B9; C; 104E1; # OSAGE CAPITAL LETTER HA
+104BA; C; 104E2; # OSAGE CAPITAL LETTER HYA
+104BB; C; 104E3; # OSAGE CAPITAL LETTER I
+104BC; C; 104E4; # OSAGE CAPITAL LETTER KA
+104BD; C; 104E5; # OSAGE CAPITAL LETTER EHKA
+104BE; C; 104E6; # OSAGE CAPITAL LETTER KYA
+104BF; C; 104E7; # OSAGE CAPITAL LETTER LA
+104C0; C; 104E8; # OSAGE CAPITAL LETTER MA
+104C1; C; 104E9; # OSAGE CAPITAL LETTER NA
+104C2; C; 104EA; # OSAGE CAPITAL LETTER O
+104C3; C; 104EB; # OSAGE CAPITAL LETTER OIN
+104C4; C; 104EC; # OSAGE CAPITAL LETTER PA
+104C5; C; 104ED; # OSAGE CAPITAL LETTER EHPA
+104C6; C; 104EE; # OSAGE CAPITAL LETTER SA
+104C7; C; 104EF; # OSAGE CAPITAL LETTER SHA
+104C8; C; 104F0; # OSAGE CAPITAL LETTER TA
+104C9; C; 104F1; # OSAGE CAPITAL LETTER EHTA
+104CA; C; 104F2; # OSAGE CAPITAL LETTER TSA
+104CB; C; 104F3; # OSAGE CAPITAL LETTER EHTSA
+104CC; C; 104F4; # OSAGE CAPITAL LETTER TSHA
+104CD; C; 104F5; # OSAGE CAPITAL LETTER DHA
+104CE; C; 104F6; # OSAGE CAPITAL LETTER U
+104CF; C; 104F7; # OSAGE CAPITAL LETTER WA
+104D0; C; 104F8; # OSAGE CAPITAL LETTER KHA
+104D1; C; 104F9; # OSAGE CAPITAL LETTER GHA
+104D2; C; 104FA; # OSAGE CAPITAL LETTER ZA
+104D3; C; 104FB; # OSAGE CAPITAL LETTER ZHA
+10570; C; 10597; # VITHKUQI CAPITAL LETTER A
+10571; C; 10598; # VITHKUQI CAPITAL LETTER BBE
+10572; C; 10599; # VITHKUQI CAPITAL LETTER BE
+10573; C; 1059A; # VITHKUQI CAPITAL LETTER CE
+10574; C; 1059B; # VITHKUQI CAPITAL LETTER CHE
+10575; C; 1059C; # VITHKUQI CAPITAL LETTER DE
+10576; C; 1059D; # VITHKUQI CAPITAL LETTER DHE
+10577; C; 1059E; # VITHKUQI CAPITAL LETTER EI
+10578; C; 1059F; # VITHKUQI CAPITAL LETTER E
+10579; C; 105A0; # VITHKUQI CAPITAL LETTER FE
+1057A; C; 105A1; # VITHKUQI CAPITAL LETTER GA
+1057C; C; 105A3; # VITHKUQI CAPITAL LETTER HA
+1057D; C; 105A4; # VITHKUQI CAPITAL LETTER HHA
+1057E; C; 105A5; # VITHKUQI CAPITAL LETTER I
+1057F; C; 105A6; # VITHKUQI CAPITAL LETTER IJE
+10580; C; 105A7; # VITHKUQI CAPITAL LETTER JE
+10581; C; 105A8; # VITHKUQI CAPITAL LETTER KA
+10582; C; 105A9; # VITHKUQI CAPITAL LETTER LA
+10583; C; 105AA; # VITHKUQI CAPITAL LETTER LLA
+10584; C; 105AB; # VITHKUQI CAPITAL LETTER ME
+10585; C; 105AC; # VITHKUQI CAPITAL LETTER NE
+10586; C; 105AD; # VITHKUQI CAPITAL LETTER NJE
+10587; C; 105AE; # VITHKUQI CAPITAL LETTER O
+10588; C; 105AF; # VITHKUQI CAPITAL LETTER PE
+10589; C; 105B0; # VITHKUQI CAPITAL LETTER QA
+1058A; C; 105B1; # VITHKUQI CAPITAL LETTER RE
+1058C; C; 105B3; # VITHKUQI CAPITAL LETTER SE
+1058D; C; 105B4; # VITHKUQI CAPITAL LETTER SHE
+1058E; C; 105B5; # VITHKUQI CAPITAL LETTER TE
+1058F; C; 105B6; # VITHKUQI CAPITAL LETTER THE
+10590; C; 105B7; # VITHKUQI CAPITAL LETTER U
+10591; C; 105B8; # VITHKUQI CAPITAL LETTER VE
+10592; C; 105B9; # VITHKUQI CAPITAL LETTER XE
+10594; C; 105BB; # VITHKUQI CAPITAL LETTER Y
+10595; C; 105BC; # VITHKUQI CAPITAL LETTER ZE
+10C80; C; 10CC0; # OLD HUNGARIAN CAPITAL LETTER A
+10C81; C; 10CC1; # OLD HUNGARIAN CAPITAL LETTER AA
+10C82; C; 10CC2; # OLD HUNGARIAN CAPITAL LETTER EB
+10C83; C; 10CC3; # OLD HUNGARIAN CAPITAL LETTER AMB
+10C84; C; 10CC4; # OLD HUNGARIAN CAPITAL LETTER EC
+10C85; C; 10CC5; # OLD HUNGARIAN CAPITAL LETTER ENC
+10C86; C; 10CC6; # OLD HUNGARIAN CAPITAL LETTER ECS
+10C87; C; 10CC7; # OLD HUNGARIAN CAPITAL LETTER ED
+10C88; C; 10CC8; # OLD HUNGARIAN CAPITAL LETTER AND
+10C89; C; 10CC9; # OLD HUNGARIAN CAPITAL LETTER E
+10C8A; C; 10CCA; # OLD HUNGARIAN CAPITAL LETTER CLOSE E
+10C8B; C; 10CCB; # OLD HUNGARIAN CAPITAL LETTER EE
+10C8C; C; 10CCC; # OLD HUNGARIAN CAPITAL LETTER EF
+10C8D; C; 10CCD; # OLD HUNGARIAN CAPITAL LETTER EG
+10C8E; C; 10CCE; # OLD HUNGARIAN CAPITAL LETTER EGY
+10C8F; C; 10CCF; # OLD HUNGARIAN CAPITAL LETTER EH
+10C90; C; 10CD0; # OLD HUNGARIAN CAPITAL LETTER I
+10C91; C; 10CD1; # OLD HUNGARIAN CAPITAL LETTER II
+10C92; C; 10CD2; # OLD HUNGARIAN CAPITAL LETTER EJ
+10C93; C; 10CD3; # OLD HUNGARIAN CAPITAL LETTER EK
+10C94; C; 10CD4; # OLD HUNGARIAN CAPITAL LETTER AK
+10C95; C; 10CD5; # OLD HUNGARIAN CAPITAL LETTER UNK
+10C96; C; 10CD6; # OLD HUNGARIAN CAPITAL LETTER EL
+10C97; C; 10CD7; # OLD HUNGARIAN CAPITAL LETTER ELY
+10C98; C; 10CD8; # OLD HUNGARIAN CAPITAL LETTER EM
+10C99; C; 10CD9; # OLD HUNGARIAN CAPITAL LETTER EN
+10C9A; C; 10CDA; # OLD HUNGARIAN CAPITAL LETTER ENY
+10C9B; C; 10CDB; # OLD HUNGARIAN CAPITAL LETTER O
+10C9C; C; 10CDC; # OLD HUNGARIAN CAPITAL LETTER OO
+10C9D; C; 10CDD; # OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG OE
+10C9E; C; 10CDE; # OLD HUNGARIAN CAPITAL LETTER RUDIMENTA OE
+10C9F; C; 10CDF; # OLD HUNGARIAN CAPITAL LETTER OEE
+10CA0; C; 10CE0; # OLD HUNGARIAN CAPITAL LETTER EP
+10CA1; C; 10CE1; # OLD HUNGARIAN CAPITAL LETTER EMP
+10CA2; C; 10CE2; # OLD HUNGARIAN CAPITAL LETTER ER
+10CA3; C; 10CE3; # OLD HUNGARIAN CAPITAL LETTER SHORT ER
+10CA4; C; 10CE4; # OLD HUNGARIAN CAPITAL LETTER ES
+10CA5; C; 10CE5; # OLD HUNGARIAN CAPITAL LETTER ESZ
+10CA6; C; 10CE6; # OLD HUNGARIAN CAPITAL LETTER ET
+10CA7; C; 10CE7; # OLD HUNGARIAN CAPITAL LETTER ENT
+10CA8; C; 10CE8; # OLD HUNGARIAN CAPITAL LETTER ETY
+10CA9; C; 10CE9; # OLD HUNGARIAN CAPITAL LETTER ECH
+10CAA; C; 10CEA; # OLD HUNGARIAN CAPITAL LETTER U
+10CAB; C; 10CEB; # OLD HUNGARIAN CAPITAL LETTER UU
+10CAC; C; 10CEC; # OLD HUNGARIAN CAPITAL LETTER NIKOLSBURG UE
+10CAD; C; 10CED; # OLD HUNGARIAN CAPITAL LETTER RUDIMENTA UE
+10CAE; C; 10CEE; # OLD HUNGARIAN CAPITAL LETTER EV
+10CAF; C; 10CEF; # OLD HUNGARIAN CAPITAL LETTER EZ
+10CB0; C; 10CF0; # OLD HUNGARIAN CAPITAL LETTER EZS
+10CB1; C; 10CF1; # OLD HUNGARIAN CAPITAL LETTER ENT-SHAPED SIGN
+10CB2; C; 10CF2; # OLD HUNGARIAN CAPITAL LETTER US
+118A0; C; 118C0; # WARANG CITI CAPITAL LETTER NGAA
+118A1; C; 118C1; # WARANG CITI CAPITAL LETTER A
+118A2; C; 118C2; # WARANG CITI CAPITAL LETTER WI
+118A3; C; 118C3; # WARANG CITI CAPITAL LETTER YU
+118A4; C; 118C4; # WARANG CITI CAPITAL LETTER YA
+118A5; C; 118C5; # WARANG CITI CAPITAL LETTER YO
+118A6; C; 118C6; # WARANG CITI CAPITAL LETTER II
+118A7; C; 118C7; # WARANG CITI CAPITAL LETTER UU
+118A8; C; 118C8; # WARANG CITI CAPITAL LETTER E
+118A9; C; 118C9; # WARANG CITI CAPITAL LETTER O
+118AA; C; 118CA; # WARANG CITI CAPITAL LETTER ANG
+118AB; C; 118CB; # WARANG CITI CAPITAL LETTER GA
+118AC; C; 118CC; # WARANG CITI CAPITAL LETTER KO
+118AD; C; 118CD; # WARANG CITI CAPITAL LETTER ENY
+118AE; C; 118CE; # WARANG CITI CAPITAL LETTER YUJ
+118AF; C; 118CF; # WARANG CITI CAPITAL LETTER UC
+118B0; C; 118D0; # WARANG CITI CAPITAL LETTER ENN
+118B1; C; 118D1; # WARANG CITI CAPITAL LETTER ODD
+118B2; C; 118D2; # WARANG CITI CAPITAL LETTER TTE
+118B3; C; 118D3; # WARANG CITI CAPITAL LETTER NUNG
+118B4; C; 118D4; # WARANG CITI CAPITAL LETTER DA
+118B5; C; 118D5; # WARANG CITI CAPITAL LETTER AT
+118B6; C; 118D6; # WARANG CITI CAPITAL LETTER AM
+118B7; C; 118D7; # WARANG CITI CAPITAL LETTER BU
+118B8; C; 118D8; # WARANG CITI CAPITAL LETTER PU
+118B9; C; 118D9; # WARANG CITI CAPITAL LETTER HIYO
+118BA; C; 118DA; # WARANG CITI CAPITAL LETTER HOLO
+118BB; C; 118DB; # WARANG CITI CAPITAL LETTER HORR
+118BC; C; 118DC; # WARANG CITI CAPITAL LETTER HAR
+118BD; C; 118DD; # WARANG CITI CAPITAL LETTER SSUU
+118BE; C; 118DE; # WARANG CITI CAPITAL LETTER SII
+118BF; C; 118DF; # WARANG CITI CAPITAL LETTER VIYO
+16E40; C; 16E60; # MEDEFAIDRIN CAPITAL LETTER M
+16E41; C; 16E61; # MEDEFAIDRIN CAPITAL LETTER S
+16E42; C; 16E62; # MEDEFAIDRIN CAPITAL LETTER V
+16E43; C; 16E63; # MEDEFAIDRIN CAPITAL LETTER W
+16E44; C; 16E64; # MEDEFAIDRIN CAPITAL LETTER ATIU
+16E45; C; 16E65; # MEDEFAIDRIN CAPITAL LETTER Z
+16E46; C; 16E66; # MEDEFAIDRIN CAPITAL LETTER KP
+16E47; C; 16E67; # MEDEFAIDRIN CAPITAL LETTER P
+16E48; C; 16E68; # MEDEFAIDRIN CAPITAL LETTER T
+16E49; C; 16E69; # MEDEFAIDRIN CAPITAL LETTER G
+16E4A; C; 16E6A; # MEDEFAIDRIN CAPITAL LETTER F
+16E4B; C; 16E6B; # MEDEFAIDRIN CAPITAL LETTER I
+16E4C; C; 16E6C; # MEDEFAIDRIN CAPITAL LETTER K
+16E4D; C; 16E6D; # MEDEFAIDRIN CAPITAL LETTER A
+16E4E; C; 16E6E; # MEDEFAIDRIN CAPITAL LETTER J
+16E4F; C; 16E6F; # MEDEFAIDRIN CAPITAL LETTER E
+16E50; C; 16E70; # MEDEFAIDRIN CAPITAL LETTER B
+16E51; C; 16E71; # MEDEFAIDRIN CAPITAL LETTER C
+16E52; C; 16E72; # MEDEFAIDRIN CAPITAL LETTER U
+16E53; C; 16E73; # MEDEFAIDRIN CAPITAL LETTER YU
+16E54; C; 16E74; # MEDEFAIDRIN CAPITAL LETTER L
+16E55; C; 16E75; # MEDEFAIDRIN CAPITAL LETTER Q
+16E56; C; 16E76; # MEDEFAIDRIN CAPITAL LETTER HP
+16E57; C; 16E77; # MEDEFAIDRIN CAPITAL LETTER NY
+16E58; C; 16E78; # MEDEFAIDRIN CAPITAL LETTER X
+16E59; C; 16E79; # MEDEFAIDRIN CAPITAL LETTER D
+16E5A; C; 16E7A; # MEDEFAIDRIN CAPITAL LETTER OE
+16E5B; C; 16E7B; # MEDEFAIDRIN CAPITAL LETTER N
+16E5C; C; 16E7C; # MEDEFAIDRIN CAPITAL LETTER R
+16E5D; C; 16E7D; # MEDEFAIDRIN CAPITAL LETTER O
+16E5E; C; 16E7E; # MEDEFAIDRIN CAPITAL LETTER AI
+16E5F; C; 16E7F; # MEDEFAIDRIN CAPITAL LETTER Y
+1E900; C; 1E922; # ADLAM CAPITAL LETTER ALIF
+1E901; C; 1E923; # ADLAM CAPITAL LETTER DAALI
+1E902; C; 1E924; # ADLAM CAPITAL LETTER LAAM
+1E903; C; 1E925; # ADLAM CAPITAL LETTER MIIM
+1E904; C; 1E926; # ADLAM CAPITAL LETTER BA
+1E905; C; 1E927; # ADLAM CAPITAL LETTER SINNYIIYHE
+1E906; C; 1E928; # ADLAM CAPITAL LETTER PE
+1E907; C; 1E929; # ADLAM CAPITAL LETTER BHE
+1E908; C; 1E92A; # ADLAM CAPITAL LETTER RA
+1E909; C; 1E92B; # ADLAM CAPITAL LETTER E
+1E90A; C; 1E92C; # ADLAM CAPITAL LETTER FA
+1E90B; C; 1E92D; # ADLAM CAPITAL LETTER I
+1E90C; C; 1E92E; # ADLAM CAPITAL LETTER O
+1E90D; C; 1E92F; # ADLAM CAPITAL LETTER DHA
+1E90E; C; 1E930; # ADLAM CAPITAL LETTER YHE
+1E90F; C; 1E931; # ADLAM CAPITAL LETTER WAW
+1E910; C; 1E932; # ADLAM CAPITAL LETTER NUN
+1E911; C; 1E933; # ADLAM CAPITAL LETTER KAF
+1E912; C; 1E934; # ADLAM CAPITAL LETTER YA
+1E913; C; 1E935; # ADLAM CAPITAL LETTER U
+1E914; C; 1E936; # ADLAM CAPITAL LETTER JIIM
+1E915; C; 1E937; # ADLAM CAPITAL LETTER CHI
+1E916; C; 1E938; # ADLAM CAPITAL LETTER HA
+1E917; C; 1E939; # ADLAM CAPITAL LETTER QAAF
+1E918; C; 1E93A; # ADLAM CAPITAL LETTER GA
+1E919; C; 1E93B; # ADLAM CAPITAL LETTER NYA
+1E91A; C; 1E93C; # ADLAM CAPITAL LETTER TU
+1E91B; C; 1E93D; # ADLAM CAPITAL LETTER NHA
+1E91C; C; 1E93E; # ADLAM CAPITAL LETTER VA
+1E91D; C; 1E93F; # ADLAM CAPITAL LETTER KHA
+1E91E; C; 1E940; # ADLAM CAPITAL LETTER GBE
+1E91F; C; 1E941; # ADLAM CAPITAL LETTER ZAL
+1E920; C; 1E942; # ADLAM CAPITAL LETTER KPO
+1E921; C; 1E943; # ADLAM CAPITAL LETTER SHA
+'''
+_map = dict(
+    (unichr(int(from_, 16)), ''.join((unichr(int(v, 16)) for v in to_.split(' '))))
+    for from_, type_, to_, _ in (
+        l.split('; ', 3) for l in _map_str.splitlines() if l)
+    if type_ in ('C', 'F'))
+del _map_str
+
+def casefold(s):
+    assert isinstance(s, compat_str)
+    return ''.join((_map.get(c, c) for c in s))
+
+__all__ = [
+    casefold
+]

From f102e3dc4efe27e6c8697d6d117f05d1bb6d1b91 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 31 Oct 2022 21:27:14 +0000
Subject: [PATCH 559/860] [compat] Add compat_casefold and compat_re_Match, for
 traverse_obj() port

---
 youtube_dl/compat.py | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index eca6d63de..4b5e1f6ed 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2375,8 +2375,10 @@ except ImportError:
 
 try:
     compat_str = unicode  # Python 2
+    from .casefold import casefold as compat_casefold
 except NameError:
     compat_str = str
+    compat_casefold = lambda s: s.casefold()
 
 try:
     from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
@@ -3066,6 +3068,9 @@ except ImportError:
 
 # Pythons disagree on the type of a pattern (RegexObject, _sre.SRE_Pattern, Pattern, ...?)
 compat_re_Pattern = type(re.compile(''))
+# and on the type of a match
+compat_re_Match = type(re.match('a', 'a'))
+
 
 if sys.version_info < (3, 3):
     def compat_b64decode(s, *args, **kwargs):
@@ -3101,6 +3106,7 @@ __all__ = [
     'compat_Struct',
     'compat_b64decode',
     'compat_basestring',
+    'compat_casefold',
     'compat_chr',
     'compat_collections_abc',
     'compat_collections_chain_map',
@@ -3132,6 +3138,7 @@ __all__ = [
     'compat_os_name',
     'compat_parse_qs',
     'compat_print',
+    'compat_re_Match',
     'compat_re_Pattern',
     'compat_realpath',
     'compat_setenv',

From b7c25959f0f76aad4ee24e254f82e6c5cca2c1ff Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 1 Nov 2022 12:40:23 +0000
Subject: [PATCH 560/860] [compat] Unify unicode/str compat and move up

---
 youtube_dl/compat.py | 31 +++++++++++++------------------
 1 file changed, 13 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 4b5e1f6ed..28942a8c1 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -21,6 +21,19 @@ import subprocess
 import sys
 import xml.etree.ElementTree
 
+# deal with critical unicode/str things first
+try:
+    # Python 2
+    compat_str, compat_basestring, compat_chr = (
+        unicode, basestring, unichr
+    )
+    from .casefold import casefold as compat_casefold
+except NameError:
+    compat_str, compat_basestring, compat_chr = (
+        str, str, chr
+    )
+    compat_casefold = lambda s: s.casefold()
+
 try:
     import collections.abc as compat_collections_abc
 except ImportError:
@@ -2373,13 +2386,6 @@ try:
 except ImportError:
     import BaseHTTPServer as compat_http_server
 
-try:
-    compat_str = unicode  # Python 2
-    from .casefold import casefold as compat_casefold
-except NameError:
-    compat_str = str
-    compat_casefold = lambda s: s.casefold()
-
 try:
     from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
     from urllib.parse import unquote as compat_urllib_parse_unquote
@@ -2510,22 +2516,11 @@ except ImportError:  # Python < 3.4
 
             return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)
 
-try:
-    compat_basestring = basestring  # Python 2
-except NameError:
-    compat_basestring = str
-
-try:
-    compat_chr = unichr  # Python 2
-except NameError:
-    compat_chr = chr
-
 try:
     from xml.etree.ElementTree import ParseError as compat_xml_parse_error
 except ImportError:  # Python 2.6
     from xml.parsers.expat import ExpatError as compat_xml_parse_error
 
-
 etree = xml.etree.ElementTree
 
 

From a874871801b8b05d06e8ffe52bed94fdfc26611e Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 1 Nov 2022 19:25:59 +0000
Subject: [PATCH 561/860] [compat] Reformat casefold.py for easier updating

---
 youtube_dl/casefold.py | 24 ++++++++++++++++++++----
 1 file changed, 20 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/casefold.py b/youtube_dl/casefold.py
index 546269a3c..7e91c3811 100644
--- a/youtube_dl/casefold.py
+++ b/youtube_dl/casefold.py
@@ -1,8 +1,20 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
-from .compat import compat_str
+from .compat import (
+    compat_str,
+    compat_chr,
+)
 
+# Below is included the text of icu/CaseFolding.txt retrieved from
+# https://github.com/unicode-org/icu/blob/main/icu4c/source/data/unidata/CaseFolding.txt
+# In case newly foldable Unicode characters are defined, paste the new version
+# of the text inside the ''' marks.
+# The text is expected to have only blank lines andlines with 1st character #,
+# all ignored, and fold definitions like this:
+# `from_hex_code; space_separated_to_hex_code_list; comment`
+
+_map_str = '''
 # CaseFolding-15.0.0.txt
 # Date: 2022-02-02, 23:35:35 GMT
 # © 2022 Unicode®, Inc.
@@ -65,7 +77,6 @@ from .compat import compat_str
 #  have the value C for the status field, and the code point itself for the mapping field.
 
 # =================================================================
-_map_str = '''
 0041; C; 0061; # LATIN CAPITAL LETTER A
 0042; C; 0062; # LATIN CAPITAL LETTER B
 0043; C; 0063; # LATIN CAPITAL LETTER C
@@ -1627,17 +1638,22 @@ FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z
 1E920; C; 1E942; # ADLAM CAPITAL LETTER KPO
 1E921; C; 1E943; # ADLAM CAPITAL LETTER SHA
 '''
+
+_parse_unichr = lambda s: compat_chr(int(s, 16))
+
 _map = dict(
-    (unichr(int(from_, 16)), ''.join((unichr(int(v, 16)) for v in to_.split(' '))))
+    (_parse_unichr(from_), ''.join(map(_parse_unichr, to_.split(' '))))
     for from_, type_, to_, _ in (
-        l.split('; ', 3) for l in _map_str.splitlines() if l)
+        l.split('; ', 3) for l in _map_str.splitlines() if l and not l[0] == '#')
     if type_ in ('C', 'F'))
 del _map_str
 
+
 def casefold(s):
     assert isinstance(s, compat_str)
     return ''.join((_map.get(c, c) for c in s))
 
+
 __all__ = [
     casefold
 ]

From 65ccb0dd4eb52cced7d0e11af021c09dbe2aed4a Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 1 Nov 2022 21:33:39 +0000
Subject: [PATCH 562/860] [compat] Add test for compat_casefold()

---
 test/test_compat.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 86ff389fd..05995372a 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -118,9 +118,21 @@ class TestCompat(unittest.TestCase):
 <smil xmlns="http://www.w3.org/2001/SMIL20/Language"></smil>'''
         compat_etree_fromstring(xml)
 
-    def test_struct_unpack(self):
+    def test_compat_struct_unpack(self):
         self.assertEqual(compat_struct_unpack('!B', b'\x00'), (0,))
 
+    def test_compat_casefold(self):
+        if hasattr(compat_str, 'casefold'):
+            # don't bother to test str.casefold() (again)
+            return
+        # thanks https://bugs.python.org/file24232/casefolding.patch
+        self.assertEqual(compat_casefold('hello'), 'hello')
+        self.assertEqual(compat_casefold('hELlo'), 'hello')
+        self.assertEqual(compat_casefold('ß'), 'ss')
+        self.assertEqual(compat_casefold('fi'), 'fi')
+        self.assertEqual(compat_casefold('\u03a3'), '\u03c3')
+        self.assertEqual(compat_casefold('A\u0345\u03a3'), 'a\u03b9\u03c3')
+
 
 if __name__ == '__main__':
     unittest.main()

From 087ddc237132103859cc00183d8d70bd75c0e44e Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 1 Nov 2022 22:47:02 +0000
Subject: [PATCH 563/860] [compat] Add test for compat_casefold()

---
 test/test_compat.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/test/test_compat.py b/test/test_compat.py
index 05995372a..0986cff37 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -11,6 +11,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 
 from youtube_dl.compat import (
+    compat_casefold,
     compat_getenv,
     compat_setenv,
     compat_etree_Element,

From c4b19a88169fa76c5eb665d274e7270a0fe452c4 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 2 Nov 2022 11:56:26 +0000
Subject: [PATCH 564/860] [compat] Work around in case folding for narrow
 Python build

Resolves #31324.
---
 youtube_dl/casefold.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/casefold.py b/youtube_dl/casefold.py
index 7e91c3811..748c2d491 100644
--- a/youtube_dl/casefold.py
+++ b/youtube_dl/casefold.py
@@ -1639,7 +1639,15 @@ FF3A; C; FF5A; # FULLWIDTH LATIN CAPITAL LETTER Z
 1E921; C; 1E943; # ADLAM CAPITAL LETTER SHA
 '''
 
-_parse_unichr = lambda s: compat_chr(int(s, 16))
+
+def _parse_unichr(s):
+    s = int(s, 16)
+    try:
+        return compat_chr(s)
+    except ValueError:
+        # work around "unichr() arg not in range(0x10000) (narrow Python build)"
+        return ('\\U%08x' % s).decode('unicode-escape')
+
 
 _map = dict(
     (_parse_unichr(from_), ''.join(map(_parse_unichr, to_.split(' '))))

From 27ed77aabba8c9eb08d66f34092b1bfcc22c482e Mon Sep 17 00:00:00 2001
From: Andrei Lebedev <lebdron@gmail.com>
Date: Thu, 3 Nov 2022 11:09:37 +0100
Subject: [PATCH 565/860] [utils] Backport traverse_obj (etc) from yt-dlp
 (#31156)

* Backport traverse_obj and closely related function from yt-dlp (code by pukkandan)
* Backport LazyList, variadic(), try_call (code by pukkandan)
* Recast using yt-dlp's newer traverse_obj() implementation and tests (code by grub4k)
* Add tests for Unicode case folding support matching Py3.5+ (requires f102e3d)
* Improve/add tests for variadic, try_call, join_nonempty

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 test/test_utils.py  | 323 +++++++++++++++++++++++++++++++++++++++++
 youtube_dl/utils.py | 339 ++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 662 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index f1a748dde..9d364c863 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -12,7 +12,9 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 # Various small unit tests
 import io
+import itertools
 import json
+import re
 import xml.etree.ElementTree
 
 from youtube_dl.utils import (
@@ -40,11 +42,14 @@ from youtube_dl.utils import (
     get_element_by_attribute,
     get_elements_by_class,
     get_elements_by_attribute,
+    get_first,
     InAdvancePagedList,
     int_or_none,
     intlist_to_bytes,
     is_html,
+    join_nonempty,
     js_to_json,
+    LazyList,
     limit_length,
     merge_dicts,
     mimetype2ext,
@@ -79,6 +84,8 @@ from youtube_dl.utils import (
     strip_or_none,
     subtitles_filename,
     timeconvert,
+    traverse_obj,
+    try_call,
     unescapeHTML,
     unified_strdate,
     unified_timestamp,
@@ -92,6 +99,7 @@ from youtube_dl.utils import (
     urlencode_postdata,
     urshift,
     update_url_query,
+    variadic,
     version_tuple,
     xpath_with_ns,
     xpath_element,
@@ -112,12 +120,18 @@ from youtube_dl.compat import (
     compat_getenv,
     compat_os_name,
     compat_setenv,
+    compat_str,
     compat_urlparse,
     compat_parse_qs,
 )
 
 
 class TestUtil(unittest.TestCase):
+
+    # yt-dlp shim
+    def assertCountEqual(self, expected, got, msg='count should be the same'):
+        return self.assertEqual(len(tuple(expected)), len(tuple(got)), msg=msg)
+
     def test_timeconvert(self):
         self.assertTrue(timeconvert('') is None)
         self.assertTrue(timeconvert('bougrg') is None)
@@ -1478,6 +1492,315 @@ Line 1
         self.assertEqual(clean_podcast_url('https://www.podtrac.com/pts/redirect.mp3/chtbl.com/track/5899E/traffic.megaphone.fm/HSW7835899191.mp3'), 'https://traffic.megaphone.fm/HSW7835899191.mp3')
         self.assertEqual(clean_podcast_url('https://play.podtrac.com/npr-344098539/edge1.pod.npr.org/anon.npr-podcasts/podcast/npr/waitwait/2020/10/20201003_waitwait_wwdtmpodcast201003-015621a5-f035-4eca-a9a1-7c118d90bc3c.mp3'), 'https://edge1.pod.npr.org/anon.npr-podcasts/podcast/npr/waitwait/2020/10/20201003_waitwait_wwdtmpodcast201003-015621a5-f035-4eca-a9a1-7c118d90bc3c.mp3')
 
+    def test_LazyList(self):
+        it = list(range(10))
+
+        self.assertEqual(list(LazyList(it)), it)
+        self.assertEqual(LazyList(it).exhaust(), it)
+        self.assertEqual(LazyList(it)[5], it[5])
+
+        self.assertEqual(LazyList(it)[5:], it[5:])
+        self.assertEqual(LazyList(it)[:5], it[:5])
+        self.assertEqual(LazyList(it)[::2], it[::2])
+        self.assertEqual(LazyList(it)[1::2], it[1::2])
+        self.assertEqual(LazyList(it)[5::-1], it[5::-1])
+        self.assertEqual(LazyList(it)[6:2:-2], it[6:2:-2])
+        self.assertEqual(LazyList(it)[::-1], it[::-1])
+
+        self.assertTrue(LazyList(it))
+        self.assertFalse(LazyList(range(0)))
+        self.assertEqual(len(LazyList(it)), len(it))
+        self.assertEqual(repr(LazyList(it)), repr(it))
+        self.assertEqual(compat_str(LazyList(it)), compat_str(it))
+
+        self.assertEqual(list(LazyList(it, reverse=True)), it[::-1])
+        self.assertEqual(list(reversed(LazyList(it))[::-1]), it)
+        self.assertEqual(list(reversed(LazyList(it))[1:3:7]), it[::-1][1:3:7])
+
+    def test_LazyList_laziness(self):
+
+        def test(ll, idx, val, cache):
+            self.assertEqual(ll[idx], val)
+            self.assertEqual(ll._cache, list(cache))
+
+        ll = LazyList(range(10))
+        test(ll, 0, 0, range(1))
+        test(ll, 5, 5, range(6))
+        test(ll, -3, 7, range(10))
+
+        ll = LazyList(range(10), reverse=True)
+        test(ll, -1, 0, range(1))
+        test(ll, 3, 6, range(10))
+
+        ll = LazyList(itertools.count())
+        test(ll, 10, 10, range(11))
+        ll = reversed(ll)
+        test(ll, -15, 14, range(15))
+
+    def test_try_call(self):
+        def total(*x, **kwargs):
+            return sum(x) + sum(kwargs.values())
+
+        self.assertEqual(try_call(None), None,
+                         msg='not a fn should give None')
+        self.assertEqual(try_call(lambda: 1), 1,
+                         msg='int fn with no expected_type should give int')
+        self.assertEqual(try_call(lambda: 1, expected_type=int), 1,
+                         msg='int fn with expected_type int should give int')
+        self.assertEqual(try_call(lambda: 1, expected_type=dict), None,
+                         msg='int fn with wrong expected_type should give None')
+        self.assertEqual(try_call(total, args=(0, 1, 0, ), expected_type=int), 1,
+                         msg='fn should accept arglist')
+        self.assertEqual(try_call(total, kwargs={'a': 0, 'b': 1, 'c': 0}, expected_type=int), 1,
+                         msg='fn should accept kwargs')
+        self.assertEqual(try_call(lambda: 1, expected_type=dict), None,
+                         msg='int fn with no expected_type should give None')
+        self.assertEqual(try_call(lambda x: {}, total, args=(42, ), expected_type=int), 42,
+                         msg='expect first int result with expected_type int')
+
+    def test_variadic(self):
+        self.assertEqual(variadic(None), (None, ))
+        self.assertEqual(variadic('spam'), ('spam', ))
+        self.assertEqual(variadic('spam', allowed_types=dict), 'spam')
+
+    def test_traverse_obj(self):
+        _TEST_DATA = {
+            100: 100,
+            1.2: 1.2,
+            'str': 'str',
+            'None': None,
+            '...': Ellipsis,
+            'urls': [
+                {'index': 0, 'url': 'https://www.example.com/0'},
+                {'index': 1, 'url': 'https://www.example.com/1'},
+            ],
+            'data': (
+                {'index': 2},
+                {'index': 3},
+            ),
+            'dict': {},
+        }
+
+        # Test base functionality
+        self.assertEqual(traverse_obj(_TEST_DATA, ('str',)), 'str',
+                         msg='allow tuple path')
+        self.assertEqual(traverse_obj(_TEST_DATA, ['str']), 'str',
+                         msg='allow list path')
+        self.assertEqual(traverse_obj(_TEST_DATA, (value for value in ("str",))), 'str',
+                         msg='allow iterable path')
+        self.assertEqual(traverse_obj(_TEST_DATA, 'str'), 'str',
+                         msg='single items should be treated as a path')
+        self.assertEqual(traverse_obj(_TEST_DATA, None), _TEST_DATA)
+        self.assertEqual(traverse_obj(_TEST_DATA, 100), 100)
+        self.assertEqual(traverse_obj(_TEST_DATA, 1.2), 1.2)
+
+        # Test Ellipsis behavior
+        self.assertCountEqual(traverse_obj(_TEST_DATA, Ellipsis),
+                              (item for item in _TEST_DATA.values() if item is not None),
+                              msg='`...` should give all values except `None`')
+        self.assertCountEqual(traverse_obj(_TEST_DATA, ('urls', 0, Ellipsis)), _TEST_DATA['urls'][0].values(),
+                              msg='`...` selection for dicts should select all values')
+        self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, Ellipsis, 'url')),
+                         ['https://www.example.com/0', 'https://www.example.com/1'],
+                         msg='nested `...` queries should work')
+        self.assertCountEqual(traverse_obj(_TEST_DATA, (Ellipsis, Ellipsis, 'index')), range(4),
+                              msg='`...` query result should be flattened')
+
+        # Test function as key
+        self.assertEqual(traverse_obj(_TEST_DATA, lambda x, y: x == 'urls' and isinstance(y, list)),
+                         [_TEST_DATA['urls']],
+                         msg='function as query key should perform a filter based on (key, value)')
+        self.assertCountEqual(traverse_obj(_TEST_DATA, lambda _, x: isinstance(x[0], compat_str)), {'str'},
+                              msg='exceptions in the query function should be caught')
+
+        # Test alternative paths
+        self.assertEqual(traverse_obj(_TEST_DATA, 'fail', 'str'), 'str',
+                         msg='multiple `paths` should be treated as alternative paths')
+        self.assertEqual(traverse_obj(_TEST_DATA, 'str', 100), 'str',
+                         msg='alternatives should exit early')
+        self.assertEqual(traverse_obj(_TEST_DATA, 'fail', 'fail'), None,
+                         msg='alternatives should return `default` if exhausted')
+        self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, 'fail'), 100), 100,
+                         msg='alternatives should track their own branching return')
+        self.assertEqual(traverse_obj(_TEST_DATA, ('dict', Ellipsis), ('data', Ellipsis)), list(_TEST_DATA['data']),
+                         msg='alternatives on empty objects should search further')
+
+        # Test branch and path nesting
+        self.assertEqual(traverse_obj(_TEST_DATA, ('urls', (3, 0), 'url')), ['https://www.example.com/0'],
+                         msg='tuple as key should be treated as branches')
+        self.assertEqual(traverse_obj(_TEST_DATA, ('urls', [3, 0], 'url')), ['https://www.example.com/0'],
+                         msg='list as key should be treated as branches')
+        self.assertEqual(traverse_obj(_TEST_DATA, ('urls', ((1, 'fail'), (0, 'url')))), ['https://www.example.com/0'],
+                         msg='double nesting in path should be treated as paths')
+        self.assertEqual(traverse_obj(['0', [1, 2]], [(0, 1), 0]), [1],
+                         msg='do not fail early on branching')
+        self.assertCountEqual(traverse_obj(_TEST_DATA, ('urls', ((1, ('fail', 'url')), (0, 'url')))),
+                              ['https://www.example.com/0', 'https://www.example.com/1'],
+                              msg='triple nesting in path should be treated as branches')
+        self.assertEqual(traverse_obj(_TEST_DATA, ('urls', ('fail', (Ellipsis, 'url')))),
+                         ['https://www.example.com/0', 'https://www.example.com/1'],
+                         msg='ellipsis as branch path start gets flattened')
+
+        # Test dictionary as key
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2}), {0: 100, 1: 1.2},
+                         msg='dict key should result in a dict with the same keys')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', 0, 'url')}),
+                         {0: 'https://www.example.com/0'},
+                         msg='dict key should allow paths')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', (3, 0), 'url')}),
+                         {0: ['https://www.example.com/0']},
+                         msg='tuple in dict path should be treated as branches')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', ((1, 'fail'), (0, 'url')))}),
+                         {0: ['https://www.example.com/0']},
+                         msg='double nesting in dict path should be treated as paths')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', ((1, ('fail', 'url')), (0, 'url')))}),
+                         {0: ['https://www.example.com/1', 'https://www.example.com/0']},
+                         msg='triple nesting in dict path should be treated as branches')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'fail'}), {},
+                         msg='remove `None` values when dict key')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'fail'}, default=Ellipsis), {0: Ellipsis},
+                         msg='do not remove `None` values if `default`')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}), {0: {}},
+                         msg='do not remove empty values when dict key')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}, default=Ellipsis), {0: {}},
+                         msg='do not remove empty values when dict key and a default')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('dict', Ellipsis)}), {0: []},
+                         msg='if branch in dict key not successful, return `[]`')
+
+        # Testing default parameter behavior
+        _DEFAULT_DATA = {'None': None, 'int': 0, 'list': []}
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, 'fail'), None,
+                         msg='default value should be `None`')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, 'fail', 'fail', default=Ellipsis), Ellipsis,
+                         msg='chained fails should result in default')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, 'None', 'int'), 0,
+                         msg='should not short cirquit on `None`')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, 'fail', default=1), 1,
+                         msg='invalid dict key should result in `default`')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, 'None', default=1), 1,
+                         msg='`None` is a deliberate sentinel and should become `default`')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, ('list', 10)), None,
+                         msg='`IndexError` should result in `default`')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, (Ellipsis, 'fail'), default=1), 1,
+                         msg='if branched but not successful return `default` if defined, not `[]`')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, (Ellipsis, 'fail'), default=None), None,
+                         msg='if branched but not successful return `default` even if `default` is `None`')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, (Ellipsis, 'fail')), [],
+                         msg='if branched but not successful return `[]`, not `default`')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, ('list', Ellipsis)), [],
+                         msg='if branched but object is empty return `[]`, not `default`')
+
+        # Testing expected_type behavior
+        _EXPECTED_TYPE_DATA = {'str': 'str', 'int': 0}
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=compat_str), 'str',
+                         msg='accept matching `expected_type` type')
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=int), None,
+                         msg='reject non matching `expected_type` type')
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'int', expected_type=lambda x: compat_str(x)), '0',
+                         msg='transform type using type function')
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str',
+                                      expected_type=lambda _: 1 / 0), None,
+                         msg='wrap expected_type function in try_call')
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, Ellipsis, expected_type=compat_str), ['str'],
+                         msg='eliminate items that expected_type fails on')
+
+        # Test get_all behavior
+        _GET_ALL_DATA = {'key': [0, 1, 2]}
+        self.assertEqual(traverse_obj(_GET_ALL_DATA, ('key', Ellipsis), get_all=False), 0,
+                         msg='if not `get_all`, return only first matching value')
+        self.assertEqual(traverse_obj(_GET_ALL_DATA, Ellipsis, get_all=False), [0, 1, 2],
+                         msg='do not overflatten if not `get_all`')
+
+        # Test casesense behavior
+        _CASESENSE_DATA = {
+            'KeY': 'value0',
+            0: {
+                'KeY': 'value1',
+                0: {'KeY': 'value2'},
+            },
+            # FULLWIDTH LATIN CAPITAL LETTER K
+            '\uff2bey': 'value3',
+        }
+        self.assertEqual(traverse_obj(_CASESENSE_DATA, 'key'), None,
+                         msg='dict keys should be case sensitive unless `casesense`')
+        self.assertEqual(traverse_obj(_CASESENSE_DATA, 'keY',
+                                      casesense=False), 'value0',
+                         msg='allow non matching key case if `casesense`')
+        self.assertEqual(traverse_obj(_CASESENSE_DATA, '\uff4bey',  # FULLWIDTH LATIN SMALL LETTER K
+                                      casesense=False), 'value3',
+                         msg='allow non matching Unicode key case if `casesense`')
+        self.assertEqual(traverse_obj(_CASESENSE_DATA, (0, ('keY',)),
+                                      casesense=False), ['value1'],
+                         msg='allow non matching key case in branch if `casesense`')
+        self.assertEqual(traverse_obj(_CASESENSE_DATA, (0, ((0, 'keY'),)),
+                                      casesense=False), ['value2'],
+                         msg='allow non matching key case in branch path if `casesense`')
+
+        # Test traverse_string behavior
+        _TRAVERSE_STRING_DATA = {'str': 'str', 1.2: 1.2}
+        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', 0)), None,
+                         msg='do not traverse into string if not `traverse_string`')
+        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', 0),
+                                      _traverse_string=True), 's',
+                         msg='traverse into string if `traverse_string`')
+        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, (1.2, 1),
+                                      _traverse_string=True), '.',
+                         msg='traverse into converted data if `traverse_string`')
+        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', Ellipsis),
+                                      _traverse_string=True), list('str'),
+                         msg='`...` branching into string should result in list')
+        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', (0, 2)),
+                                      _traverse_string=True), ['s', 'r'],
+                         msg='branching into string should result in list')
+        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', lambda _, x: x),
+                                      _traverse_string=True), list('str'),
+                         msg='function branching into string should result in list')
+
+        # Test is_user_input behavior
+        _IS_USER_INPUT_DATA = {'range8': list(range(8))}
+        self.assertEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', '3'),
+                                      _is_user_input=True), 3,
+                         msg='allow for string indexing if `is_user_input`')
+        self.assertCountEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', '3:'),
+                                           _is_user_input=True), tuple(range(8))[3:],
+                              msg='allow for string slice if `is_user_input`')
+        self.assertCountEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', ':4:2'),
+                                           _is_user_input=True), tuple(range(8))[:4:2],
+                              msg='allow step in string slice if `is_user_input`')
+        self.assertCountEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', ':'),
+                                           _is_user_input=True), range(8),
+                              msg='`:` should be treated as `...` if `is_user_input`')
+        with self.assertRaises(TypeError, msg='too many params should result in error'):
+            traverse_obj(_IS_USER_INPUT_DATA, ('range8', ':::'), _is_user_input=True)
+
+        # Test re.Match as input obj
+        mobj = re.match(r'^0(12)(?P<group>3)(4)?$', '0123')
+        self.assertEqual(traverse_obj(mobj, Ellipsis), [x for x in mobj.groups() if x is not None],
+                         msg='`...` on a `re.Match` should give its `groups()`')
+        self.assertEqual(traverse_obj(mobj, lambda k, _: k in (0, 2)), ['0123', '3'],
+                         msg='function on a `re.Match` should give groupno, value starting at 0')
+        self.assertEqual(traverse_obj(mobj, 'group'), '3',
+                         msg='str key on a `re.Match` should give group with that name')
+        self.assertEqual(traverse_obj(mobj, 2), '3',
+                         msg='int key on a `re.Match` should give group with that name')
+        self.assertEqual(traverse_obj(mobj, 'gRoUp', casesense=False), '3',
+                         msg='str key on a `re.Match` should respect casesense')
+        self.assertEqual(traverse_obj(mobj, 'fail'), None,
+                         msg='failing str key on a `re.Match` should return `default`')
+        self.assertEqual(traverse_obj(mobj, 'gRoUpS', casesense=False), None,
+                         msg='failing str key on a `re.Match` should return `default`')
+        self.assertEqual(traverse_obj(mobj, 8), None,
+                         msg='failing int key on a `re.Match` should return `default`')
+
+    def test_get_first(self):
+        self.assertEqual(get_first([{'a': None}, {'a': 'spam'}], 'a'), 'spam')
+
+    def test_join_nonempty(self):
+        self.assertEqual(join_nonempty('a', 'b'), 'a-b')
+        self.assertEqual(join_nonempty(
+            'a', 'b', 'c', 'd',
+            from_dict={'a': 'c', 'c': [], 'b': 'd', 'd': None}), 'c-d')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 23a65a81c..e3c3ccff9 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -43,6 +43,7 @@ from .compat import (
     compat_HTTPError,
     compat_basestring,
     compat_chr,
+    compat_collections_abc,
     compat_cookiejar,
     compat_ctypes_WINFUNCTYPE,
     compat_etree_fromstring,
@@ -1685,6 +1686,7 @@ USER_AGENTS = {
 
 
 NO_DEFAULT = object()
+IDENTITY = lambda x: x
 
 ENGLISH_MONTH_NAMES = [
     'January', 'February', 'March', 'April', 'May', 'June',
@@ -3867,6 +3869,105 @@ def detect_exe_version(output, version_re=None, unrecognized='present'):
         return unrecognized
 
 
+class LazyList(compat_collections_abc.Sequence):
+    """Lazy immutable list from an iterable
+    Note that slices of a LazyList are lists and not LazyList"""
+
+    class IndexError(IndexError):
+        def __init__(self, cause=None):
+            if cause:
+                # reproduce `raise from`
+                self.__cause__ = cause
+            super(IndexError, self).__init__()
+
+    def __init__(self, iterable, **kwargs):
+        # kwarg-only
+        reverse = kwargs.get('reverse', False)
+        _cache = kwargs.get('_cache')
+
+        self._iterable = iter(iterable)
+        self._cache = [] if _cache is None else _cache
+        self._reversed = reverse
+
+    def __iter__(self):
+        if self._reversed:
+            # We need to consume the entire iterable to iterate in reverse
+            for item in self.exhaust():
+                yield item
+            return
+        for item in self._cache:
+            yield item
+        for item in self._iterable:
+            self._cache.append(item)
+            yield item
+
+    def _exhaust(self):
+        self._cache.extend(self._iterable)
+        self._iterable = []  # Discard the emptied iterable to make it pickle-able
+        return self._cache
+
+    def exhaust(self):
+        """Evaluate the entire iterable"""
+        return self._exhaust()[::-1 if self._reversed else 1]
+
+    @staticmethod
+    def _reverse_index(x):
+        return None if x is None else ~x
+
+    def __getitem__(self, idx):
+        if isinstance(idx, slice):
+            if self._reversed:
+                idx = slice(self._reverse_index(idx.start), self._reverse_index(idx.stop), -(idx.step or 1))
+            start, stop, step = idx.start, idx.stop, idx.step or 1
+        elif isinstance(idx, int):
+            if self._reversed:
+                idx = self._reverse_index(idx)
+            start, stop, step = idx, idx, 0
+        else:
+            raise TypeError('indices must be integers or slices')
+        if ((start or 0) < 0 or (stop or 0) < 0
+                or (start is None and step < 0)
+                or (stop is None and step > 0)):
+            # We need to consume the entire iterable to be able to slice from the end
+            # Obviously, never use this with infinite iterables
+            self._exhaust()
+            try:
+                return self._cache[idx]
+            except IndexError as e:
+                raise self.IndexError(e)
+        n = max(start or 0, stop or 0) - len(self._cache) + 1
+        if n > 0:
+            self._cache.extend(itertools.islice(self._iterable, n))
+        try:
+            return self._cache[idx]
+        except IndexError as e:
+            raise self.IndexError(e)
+
+    def __bool__(self):
+        try:
+            self[-1] if self._reversed else self[0]
+        except self.IndexError:
+            return False
+        return True
+
+    def __len__(self):
+        self._exhaust()
+        return len(self._cache)
+
+    def __reversed__(self):
+        return type(self)(self._iterable, reverse=not self._reversed, _cache=self._cache)
+
+    def __copy__(self):
+        return type(self)(self._iterable, reverse=self._reversed, _cache=self._cache)
+
+    def __repr__(self):
+        # repr and str should mimic a list. So we exhaust the iterable
+        return repr(self.exhaust())
+
+    def __str__(self):
+        return repr(self.exhaust())
+
+
 class PagedList(object):
     def __len__(self):
         # This is only useful for tests
@@ -4092,6 +4193,10 @@ def multipart_encode(data, boundary=None):
     return out, content_type
 
 
+def variadic(x, allowed_types=(compat_str, bytes, dict)):
+    return x if isinstance(x, compat_collections_abc.Iterable) and not isinstance(x, allowed_types) else (x,)
+
+
 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
     if isinstance(key_or_keys, (list, tuple)):
         for key in key_or_keys:
@@ -4102,6 +4207,23 @@ def dict_get(d, key_or_keys, default=None, skip_false_values=True):
     return d.get(key_or_keys, default)
 
 
+def try_call(*funcs, **kwargs):
+
+    # parameter defaults
+    expected_type = kwargs.get('expected_type')
+    fargs = kwargs.get('args', [])
+    fkwargs = kwargs.get('kwargs', {})
+
+    for f in funcs:
+        try:
+            val = f(*fargs, **fkwargs)
+        except (AttributeError, KeyError, TypeError, IndexError, ZeroDivisionError):
+            pass
+        else:
+            if expected_type is None or isinstance(val, expected_type):
+                return val
+
+
 def try_get(src, getter, expected_type=None):
     if not isinstance(getter, (list, tuple)):
         getter = [getter]
@@ -5835,3 +5957,220 @@ def clean_podcast_url(url):
                 st\.fm # https://podsights.com/docs/
             )/e
         )/''', '', url)
+
+
+def traverse_obj(obj, *paths, **kwargs):
+    """
+    Safely traverse nested `dict`s and `Sequence`s
+
+    >>> obj = [{}, {"key": "value"}]
+    >>> traverse_obj(obj, (1, "key"))
+    "value"
+
+    Each of the provided `paths` is tested and the first producing a valid result will be returned.
+    The next path will also be tested if the path branched but no results could be found.
+    Supported values for traversal are `Mapping`, `Sequence` and `re.Match`.
+    A value of None is treated as the absence of a value.
+
+    The paths will be wrapped in `variadic`, so that `'key'` is conveniently the same as `('key', )`.
+
+    The keys in the path can be one of:
+        - `None`:           Return the current object.
+        - `str`/`int`:      Return `obj[key]`. For `re.Match, return `obj.group(key)`.
+        - `slice`:          Branch out and return all values in `obj[key]`.
+        - `Ellipsis`:       Branch out and return a list of all values.
+        - `tuple`/`list`:   Branch out and return a list of all matching values.
+                            Read as: `[traverse_obj(obj, branch) for branch in branches]`.
+        - `function`:       Branch out and return values filtered by the function.
+                            Read as: `[value for key, value in obj if function(key, value)]`.
+                            For `Sequence`s, `key` is the index of the value.
+        - `dict`            Transform the current object and return a matching dict.
+                            Read as: `{key: traverse_obj(obj, path) for key, path in dct.items()}`.
+
+        `tuple`, `list`, and `dict` all support nested paths and branches.
+
+    @params paths           Paths which to traverse by.
+    Keyword arguments:
+    @param default          Value to return if the paths do not match.
+    @param expected_type    If a `type`, only accept final values of this type.
+                            If any other callable, try to call the function on each result.
+    @param get_all          If `False`, return the first matching result, otherwise all matching ones.
+    @param casesense        If `False`, consider string dictionary keys as case insensitive.
+
+    The following are only meant to be used by YoutubeDL.prepare_outtmpl and are not part of the API
+
+    @param _is_user_input    Whether the keys are generated from user input.
+                            If `True` strings get converted to `int`/`slice` if needed.
+    @param _traverse_string  Whether to traverse into objects as strings.
+                            If `True`, any non-compatible object will first be
+                            converted into a string and then traversed into.
+
+
+    @returns                The result of the object traversal.
+                            If successful, `get_all=True`, and the path branches at least once,
+                            then a list of results is returned instead.
+                            A list is always returned if the last path branches and no `default` is given.
+    """
+
+    # parameter defaults
+    default = kwargs.get('default', NO_DEFAULT)
+    expected_type = kwargs.get('expected_type')
+    get_all = kwargs.get('get_all', True)
+    casesense = kwargs.get('casesense', True)
+    _is_user_input = kwargs.get('_is_user_input', False)
+    _traverse_string = kwargs.get('_traverse_string', False)
+
+    # instant compat
+    str = compat_str
+
+    is_sequence = lambda x: isinstance(x, compat_collections_abc.Sequence) and not isinstance(x, (str, bytes))
+    # stand-in until compat_re_Match is added
+    compat_re_Match = type(re.match('a', 'a'))
+    # stand-in until casefold.py is added
+    try:
+        ''.casefold()
+        compat_casefold = lambda s: s.casefold()
+    except AttributeError:
+        compat_casefold = lambda s: s.lower()
+    casefold = lambda k: compat_casefold(k) if isinstance(k, str) else k
+
+    if isinstance(expected_type, type):
+        type_test = lambda val: val if isinstance(val, expected_type) else None
+    else:
+        type_test = lambda val: try_call(expected_type or IDENTITY, args=(val,))
+
+    def from_iterable(iterables):
+        # chain.from_iterable(['ABC', 'DEF']) --> A B C D E F
+        for it in iterables:
+            for item in it:
+                yield item
+
+    def apply_key(key, obj):
+        if obj is None:
+            return
+
+        elif key is None:
+            yield obj
+
+        elif isinstance(key, (list, tuple)):
+            for branch in key:
+                _, result = apply_path(obj, branch)
+                for item in result:
+                    yield item
+
+        elif key is Ellipsis:
+            result = []
+            if isinstance(obj, compat_collections_abc.Mapping):
+                result = obj.values()
+            elif is_sequence(obj):
+                result = obj
+            elif isinstance(obj, compat_re_Match):
+                result = obj.groups()
+            elif _traverse_string:
+                result = str(obj)
+            for item in result:
+                yield item
+
+        elif callable(key):
+            if is_sequence(obj):
+                iter_obj = enumerate(obj)
+            elif isinstance(obj, compat_collections_abc.Mapping):
+                iter_obj = obj.items()
+            elif isinstance(obj, compat_re_Match):
+                iter_obj = enumerate(itertools.chain([obj.group()], obj.groups()))
+            elif _traverse_string:
+                iter_obj = enumerate(str(obj))
+            else:
+                return
+            for item in (v for k, v in iter_obj if try_call(key, args=(k, v))):
+                yield item
+
+        elif isinstance(key, dict):
+            iter_obj = ((k, _traverse_obj(obj, v)) for k, v in key.items())
+            yield dict((k, v if v is not None else default) for k, v in iter_obj
+                       if v is not None or default is not NO_DEFAULT)
+
+        elif isinstance(obj, compat_collections_abc.Mapping):
+            yield (obj.get(key) if casesense or (key in obj)
+                   else next((v for k, v in obj.items() if casefold(k) == key), None))
+
+        elif isinstance(obj, compat_re_Match):
+            if isinstance(key, int) or casesense:
+                try:
+                    yield obj.group(key)
+                    return
+                except IndexError:
+                    pass
+            if not isinstance(key, str):
+                return
+
+            yield next((v for k, v in obj.groupdict().items() if casefold(k) == key), None)
+
+        else:
+            if _is_user_input:
+                key = (int_or_none(key) if ':' not in key
+                       else slice(*map(int_or_none, key.split(':'))))
+
+            if not isinstance(key, (int, slice)):
+                return
+
+            if not is_sequence(obj):
+                if not _traverse_string:
+                    return
+                obj = str(obj)
+
+            try:
+                yield obj[key]
+            except IndexError:
+                pass
+
+    def apply_path(start_obj, path):
+        objs = (start_obj,)
+        has_branched = False
+
+        for key in variadic(path):
+            if _is_user_input and key == ':':
+                key = Ellipsis
+
+            if not casesense and isinstance(key, str):
+                key = compat_casefold(key)
+
+            if key is Ellipsis or isinstance(key, (list, tuple)) or callable(key):
+                has_branched = True
+
+            key_func = functools.partial(apply_key, key)
+            objs = from_iterable(map(key_func, objs))
+
+        return has_branched, objs
+
+    def _traverse_obj(obj, path, use_list=True):
+        has_branched, results = apply_path(obj, path)
+        results = LazyList(x for x in map(type_test, results) if x is not None)
+
+        if get_all and has_branched:
+            return results.exhaust() if results or use_list else None
+
+        return results[0] if results else None
+
+    for index, path in enumerate(paths, 1):
+        use_list = default is NO_DEFAULT and index == len(paths)
+        result = _traverse_obj(obj, path, use_list)
+        if result is not None:
+            return result
+
+    return None if default is NO_DEFAULT else default
+
+
+def get_first(obj, keys, **kwargs):
+    return traverse_obj(obj, (Ellipsis,) + tuple(variadic(keys)), get_all=False, **kwargs)
+
+
+def join_nonempty(*values, **kwargs):
+
+    # parameter defaults
+    delim = kwargs.get('delim', '-')
+    from_dict = kwargs.get('from_dict')
+
+    if from_dict is not None:
+        values = (traverse_obj(from_dict, variadic(v)) for v in values)
+    return delim.join(map(compat_str, filter(None, values)))

From de39d1281cea499cb1adfce5ff7e0a56f1bad5fe Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Nov 2022 10:13:07 +0000
Subject: [PATCH 566/860] [extractor/ceskatelevize] Back-port extractor from
 yt-dlp, etc (#30713)

* back-port extractor, removing CeskaTelevizePoradyIE
* follow redirect URL
* support liveBroadcast and videobonusDetail in __NEXT__ data
* return single video for singleton playlist
* fix/add tests
---
 youtube_dl/extractor/ceskatelevize.py | 178 ++++++++++++++------------
 youtube_dl/extractor/extractors.py    |   5 +-
 2 files changed, 96 insertions(+), 87 deletions(-)

diff --git a/youtube_dl/extractor/ceskatelevize.py b/youtube_dl/extractor/ceskatelevize.py
index 7cb4efb74..fe677d8e8 100644
--- a/youtube_dl/extractor/ceskatelevize.py
+++ b/youtube_dl/extractor/ceskatelevize.py
@@ -12,35 +12,21 @@ from ..utils import (
     ExtractorError,
     float_or_none,
     sanitized_Request,
-    unescapeHTML,
-    update_url_query,
+    str_or_none,
+    traverse_obj,
     urlencode_postdata,
     USER_AGENTS,
 )
 
 
 class CeskaTelevizeIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/ivysilani/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
+    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/(?:ivysilani|porady|zive)/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
     _TESTS = [{
-        'url': 'http://www.ceskatelevize.cz/ivysilani/ivysilani/10441294653-hyde-park-civilizace/214411058091220',
-        'info_dict': {
-            'id': '61924494877246241',
-            'ext': 'mp4',
-            'title': 'Hyde Park Civilizace: Život v Grónsku',
-            'description': 'md5:3fec8f6bb497be5cdb0c9e8781076626',
-            'thumbnail': r're:^https?://.*\.jpg',
-            'duration': 3350,
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }, {
         'url': 'http://www.ceskatelevize.cz/ivysilani/10441294653-hyde-park-civilizace/215411058090502/bonus/20641-bonus-01-en',
         'info_dict': {
             'id': '61924494877028507',
             'ext': 'mp4',
-            'title': 'Hyde Park Civilizace: Bonus 01 - En',
+            'title': 'Bonus 01 - En - Hyde Park Civilizace',
             'description': 'English Subtittles',
             'thumbnail': r're:^https?://.*\.jpg',
             'duration': 81.3,
@@ -51,31 +37,111 @@ class CeskaTelevizeIE(InfoExtractor):
         },
     }, {
         # live stream
-        'url': 'http://www.ceskatelevize.cz/ivysilani/zive/ct4/',
+        'url': 'http://www.ceskatelevize.cz/zive/ct1/',
         'info_dict': {
-            'id': 402,
+            'id': '102',
             'ext': 'mp4',
-            'title': r're:^ČT Sport \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
+            'title': r'ČT1 - živé vysílání online',
+            'description': 'Sledujte živé vysílání kanálu ČT1 online. Vybírat si můžete i z dalších kanálů České televize na kterémkoli z vašich zařízení.',
             'is_live': True,
         },
         'params': {
             # m3u8 download
             'skip_download': True,
         },
-        'skip': 'Georestricted to Czech Republic',
+    }, {
+        # another
+        'url': 'http://www.ceskatelevize.cz/ivysilani/zive/ct4/',
+        'only_matching': True,
+        'info_dict': {
+            'id': 402,
+            'ext': 'mp4',
+            'title': r're:^ČT Sport \d{4}-\d{2}-\d{2} \d{2}:\d{2}$',
+            'is_live': True,
+        },
+        # 'skip': 'Georestricted to Czech Republic',
     }, {
         'url': 'http://www.ceskatelevize.cz/ivysilani/embed/iFramePlayer.php?hash=d6a3e1370d2e4fa76296b90bad4dfc19673b641e&IDEC=217 562 22150/0004&channelID=1&width=100%25',
         'only_matching': True,
+    }, {
+        # video with 18+ caution trailer
+        'url': 'http://www.ceskatelevize.cz/porady/10520528904-queer/215562210900007-bogotart/',
+        'info_dict': {
+            'id': '215562210900007-bogotart',
+            'title': 'Bogotart - Queer',
+            'description': 'Hlavní město Kolumbie v doprovodu queer umělců. Vroucí svět plný vášně, sebevědomí, ale i násilí a bolesti',
+        },
+        'playlist': [{
+            'info_dict': {
+                'id': '61924494877311053',
+                'ext': 'mp4',
+                'title': 'Bogotart - Queer (Varování 18+)',
+                'duration': 11.9,
+            },
+        }, {
+            'info_dict': {
+                'id': '61924494877068022',
+                'ext': 'mp4',
+                'title': 'Bogotart - Queer (Queer)',
+                'thumbnail': r're:^https?://.*\.jpg',
+                'duration': 1558.3,
+            },
+        }],
+        'params': {
+            # m3u8 download
+            'skip_download': True,
+        },
+    }, {
+        # iframe embed
+        'url': 'http://www.ceskatelevize.cz/porady/10614999031-neviditelni/21251212048/',
+        'only_matching': True,
     }]
 
+    def _search_nextjs_data(self, webpage, video_id, **kw):
+        return self._parse_json(
+            self._search_regex(
+                r'(?s)<script[^>]+id=[\'"]__NEXT_DATA__[\'"][^>]*>([^<]+)</script>',
+                webpage, 'next.js data', **kw),
+            video_id, **kw)
+
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
+        webpage, urlh = self._download_webpage_handle(url, playlist_id)
+        parsed_url = compat_urllib_parse_urlparse(urlh.geturl())
+        site_name = self._og_search_property('site_name', webpage, fatal=False, default='Česká televize')
+        playlist_title = self._og_search_title(webpage, default=None)
+        if site_name and playlist_title:
+            playlist_title = re.split(r'\s*[—|]\s*%s' % (site_name, ), playlist_title, 1)[0]
+        playlist_description = self._og_search_description(webpage, default=None)
+        if playlist_description:
+            playlist_description = playlist_description.replace('\xa0', ' ')
 
-        webpage = self._download_webpage(url, playlist_id)
+        type_ = 'IDEC'
+        if re.search(r'(^/porady|/zive)/', parsed_url.path):
+            next_data = self._search_nextjs_data(webpage, playlist_id)
+            if '/zive/' in parsed_url.path:
+                idec = traverse_obj(next_data, ('props', 'pageProps', 'data', 'liveBroadcast', 'current', 'idec'), get_all=False)
+            else:
+                idec = traverse_obj(next_data, ('props', 'pageProps', 'data', ('show', 'mediaMeta'), 'idec'), get_all=False)
+                if not idec:
+                    idec = traverse_obj(next_data, ('props', 'pageProps', 'data', 'videobonusDetail', 'bonusId'), get_all=False)
+                    if idec:
+                        type_ = 'bonus'
+            if not idec:
+                raise ExtractorError('Failed to find IDEC id')
+            iframe_hash = self._download_webpage(
+                'https://www.ceskatelevize.cz/v-api/iframe-hash/',
+                playlist_id, note='Getting IFRAME hash')
+            query = {'hash': iframe_hash, 'origin': 'iVysilani', 'autoStart': 'true', type_: idec, }
+            webpage = self._download_webpage(
+                'https://www.ceskatelevize.cz/ivysilani/embed/iFramePlayer.php',
+                playlist_id, note='Downloading player', query=query)
 
         NOT_AVAILABLE_STRING = 'This content is not available at your territory due to limited copyright.'
         if '%s</p>' % NOT_AVAILABLE_STRING in webpage:
-            raise ExtractorError(NOT_AVAILABLE_STRING, expected=True)
+            self.raise_geo_restricted(NOT_AVAILABLE_STRING)
+        if any(not_found in webpage for not_found in ('Neplatný parametr pro videopřehrávač', 'IDEC nebyl nalezen', )):
+            raise ExtractorError('no video with IDEC available', video_id=idec, expected=True)
 
         type_ = None
         episode_id = None
@@ -100,7 +166,7 @@ class CeskaTelevizeIE(InfoExtractor):
         data = {
             'playlist[0][type]': type_,
             'playlist[0][id]': episode_id,
-            'requestUrl': compat_urllib_parse_urlparse(url).path,
+            'requestUrl': parsed_url.path,
             'requestSource': 'iVysilani',
         }
 
@@ -108,7 +174,7 @@ class CeskaTelevizeIE(InfoExtractor):
 
         for user_agent in (None, USER_AGENTS['Safari']):
             req = sanitized_Request(
-                'https://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist',
+                'https://www.ceskatelevize.cz/ivysilani/ajax/get-client-playlist/',
                 data=urlencode_postdata(data))
 
             req.add_header('Content-type', 'application/x-www-form-urlencoded')
@@ -130,9 +196,6 @@ class CeskaTelevizeIE(InfoExtractor):
             req = sanitized_Request(compat_urllib_parse_unquote(playlist_url))
             req.add_header('Referer', url)
 
-            playlist_title = self._og_search_title(webpage, default=None)
-            playlist_description = self._og_search_description(webpage, default=None)
-
             playlist = self._download_json(req, playlist_id, fatal=False)
             if not playlist:
                 continue
@@ -167,7 +230,7 @@ class CeskaTelevizeIE(InfoExtractor):
                     entries[num]['formats'].extend(formats)
                     continue
 
-                item_id = item.get('id') or item['assetId']
+                item_id = str_or_none(item.get('id') or item['assetId'])
                 title = item['title']
 
                 duration = float_or_none(item.get('duration'))
@@ -181,8 +244,6 @@ class CeskaTelevizeIE(InfoExtractor):
 
                 if playlist_len == 1:
                     final_title = playlist_title or title
-                    if is_live:
-                        final_title = self._live_title(final_title)
                 else:
                     final_title = '%s (%s)' % (playlist_title, title)
 
@@ -200,6 +261,8 @@ class CeskaTelevizeIE(InfoExtractor):
         for e in entries:
             self._sort_formats(e['formats'])
 
+        if len(entries) == 1:
+            return entries[0]
         return self.playlist_result(entries, playlist_id, playlist_title, playlist_description)
 
     def _get_subtitles(self, episode_id, subs):
@@ -236,54 +299,3 @@ class CeskaTelevizeIE(InfoExtractor):
                     yield line
 
         return '\r\n'.join(_fix_subtitle(subtitles))
-
-
-class CeskaTelevizePoradyIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?ceskatelevize\.cz/porady/(?:[^/?#&]+/)*(?P<id>[^/#?]+)'
-    _TESTS = [{
-        # video with 18+ caution trailer
-        'url': 'http://www.ceskatelevize.cz/porady/10520528904-queer/215562210900007-bogotart/',
-        'info_dict': {
-            'id': '215562210900007-bogotart',
-            'title': 'Queer: Bogotart',
-            'description': 'Alternativní průvodce současným queer světem',
-        },
-        'playlist': [{
-            'info_dict': {
-                'id': '61924494876844842',
-                'ext': 'mp4',
-                'title': 'Queer: Bogotart (Varování 18+)',
-                'duration': 10.2,
-            },
-        }, {
-            'info_dict': {
-                'id': '61924494877068022',
-                'ext': 'mp4',
-                'title': 'Queer: Bogotart (Queer)',
-                'thumbnail': r're:^https?://.*\.jpg',
-                'duration': 1558.3,
-            },
-        }],
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
-    }, {
-        # iframe embed
-        'url': 'http://www.ceskatelevize.cz/porady/10614999031-neviditelni/21251212048/',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-
-        data_url = update_url_query(unescapeHTML(self._search_regex(
-            (r'<span[^>]*\bdata-url=(["\'])(?P<url>(?:(?!\1).)+)\1',
-             r'<iframe[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:www\.)?ceskatelevize\.cz/ivysilani/embed/iFramePlayer\.php.*?)\1'),
-            webpage, 'iframe player url', group='url')), query={
-                'autoStart': 'true',
-        })
-
-        return self.url_result(data_url, ie=CeskaTelevizeIE.ie_key())
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 751fc38b6..e36f86be4 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -208,10 +208,7 @@ from .ccc import (
 from .ccma import CCMAIE
 from .cctv import CCTVIE
 from .cda import CDAIE
-from .ceskatelevize import (
-    CeskaTelevizeIE,
-    CeskaTelevizePoradyIE,
-)
+from .ceskatelevize import CeskaTelevizeIE
 from .channel9 import Channel9IE
 from .charlierose import CharlieRoseIE
 from .chaturbate import ChaturbateIE

From 47e70fff8ba3de769a31fab0b3572162094733f7 Mon Sep 17 00:00:00 2001
From: Moises Lima <mozlima@users.noreply.github.com>
Date: Wed, 9 Nov 2022 17:26:30 -0300
Subject: [PATCH 567/860] [PeekVids, PlayVids] Add new extractor (#29765)

* Merge back-port from yt-dlp
* Merge features from PR #29798
* Improve metadata extraction

Co-authored-by: dirkf <fieldhouse@gmx.net>
Co-authored by: AXDOOMER
---
 youtube_dl/extractor/extractors.py |   4 +
 youtube_dl/extractor/peekvids.py   | 193 +++++++++++++++++++++++++++++
 2 files changed, 197 insertions(+)
 create mode 100644 youtube_dl/extractor/peekvids.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index e36f86be4..4d9f37424 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -909,6 +909,10 @@ from .parliamentliveuk import ParliamentLiveUKIE
 from .patreon import PatreonIE
 from .pbs import PBSIE
 from .pearvideo import PearVideoIE
+from .peekvids import (
+    PeekVidsIE,
+    PlayVidsIE,
+)
 from .peertube import PeerTubeIE
 from .people import PeopleIE
 from .performgroup import PerformGroupIE
diff --git a/youtube_dl/extractor/peekvids.py b/youtube_dl/extractor/peekvids.py
new file mode 100644
index 000000000..c8aad564b
--- /dev/null
+++ b/youtube_dl/extractor/peekvids.py
@@ -0,0 +1,193 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    ExtractorError,
+    get_element_by_class,
+    int_or_none,
+    merge_dicts,
+    url_or_none,
+)
+
+
+class PeekVidsIE(InfoExtractor):
+    _VALID_URL = r'''(?x)
+        https?://(?:www\.)?peekvids\.com/
+        (?:(?:[^/?#]+/){2}|embed/?\?(?:[^#]*&)?v=)
+        (?P<id>[^/?&#]*)
+    '''
+    _TESTS = [{
+        'url': 'https://peekvids.com/pc/dane-jones-cute-redhead-with-perfect-tits-with-mini-vamp/BSyLMbN0YCd',
+        'md5': '2ff6a357a9717dc9dc9894b51307e9a2',
+        'info_dict': {
+            'id': '1262717',
+            'display_id': 'BSyLMbN0YCd',
+            'title': ' Dane Jones - Cute redhead with perfect tits with Mini Vamp',
+            'ext': 'mp4',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': 'md5:0a61df3620de26c0af8963b1a730cd69',
+            'timestamp': 1642579329,
+            'upload_date': '20220119',
+            'duration': 416,
+            'view_count': int,
+            'age_limit': 18,
+            'uploader': 'SEXYhub.com',
+            'categories': list,
+            'tags': list,
+        },
+    }]
+    _DOMAIN = 'www.peekvids.com'
+
+    def _get_detail(self, html):
+        return get_element_by_class('detail-video-block', html)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id, expected_status=429)
+        if '>Rate Limit Exceeded' in webpage:
+            raise ExtractorError(
+                '[%s] %s: %s' % (self.IE_NAME, video_id, 'You are suspected as a bot. Wait, or pass the captcha test on the site and provide --cookies.'),
+                expected=True)
+
+        title = self._html_search_regex(r'(?s)<h1\b[^>]*>(.+?)</h1>', webpage, 'title')
+
+        display_id = video_id
+        video_id = self._search_regex(r'(?s)<video\b[^>]+\bdata-id\s*=\s*["\']?([\w-]+)', webpage, 'short video ID')
+        srcs = self._download_json(
+            'https://%s/v-alt/%s' % (self._DOMAIN, video_id), video_id,
+            note='Downloading list of source files')
+        formats = [{
+            'url': f_url,
+            'format_id': f_id,
+            'height': int_or_none(f_id),
+        } for f_url, f_id in (
+            (url_or_none(f_v), f_match.group(1))
+            for f_v, f_match in (
+                (v, re.match(r'^data-src(\d{3,})$', k))
+                for k, v in srcs.items() if v) if f_match)
+            if f_url
+        ]
+        if not formats:
+            formats = [{'url': url} for url in srcs.values()]
+        self._sort_formats(formats)
+
+        info = self._search_json_ld(webpage, video_id, expected_type='VideoObject', default={})
+        info.pop('url', None)
+        # may not have found the thumbnail if it was in a list in the ld+json
+        info.setdefault('thumbnail', self._og_search_thumbnail(webpage))
+        detail = self._get_detail(webpage) or ''
+        info['description'] = self._html_search_regex(
+            r'(?s)(.+?)(?:%s\s*<|<ul\b)' % (re.escape(info.get('description', '')), ),
+            detail, 'description', default=None) or None
+        info['title'] = re.sub(r'\s*[,-][^,-]+$', '', info.get('title') or title) or self._generic_title(url)
+
+        def cat_tags(name, html):
+            l = self._html_search_regex(
+                r'(?s)<span\b[^>]*>\s*%s\s*:\s*</span>(.+?)</li>' % (re.escape(name), ),
+                html, name, default='')
+            return [x for x in re.split(r'\s+', l) if x]
+
+        return merge_dicts({
+            'id': video_id,
+            'display_id': display_id,
+            'age_limit': 18,
+            'formats': formats,
+            'categories': cat_tags('Categories', detail),
+            'tags': cat_tags('Tags', detail),
+            'uploader': self._html_search_regex(r'[Uu]ploaded\s+by\s(.+?)"', webpage, 'uploader', default=None),
+        }, info)
+
+
+class PlayVidsIE(PeekVidsIE):
+    _VALID_URL = r'https?://(?:www\.)?playvids\.com/(?:embed/|\w\w?/)?(?P<id>[^/?#]*)'
+    _TESTS = [{
+        'url': 'https://www.playvids.com/U3pBrYhsjXM/pc/dane-jones-cute-redhead-with-perfect-tits-with-mini-vamp',
+        'md5': '2f12e50213dd65f142175da633c4564c',
+        'info_dict': {
+            'id': '1978030',
+            'display_id': 'U3pBrYhsjXM',
+            'title': ' Dane Jones - Cute redhead with perfect tits with Mini Vamp',
+            'ext': 'mp4',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'description': 'md5:0a61df3620de26c0af8963b1a730cd69',
+            'timestamp': 1640435839,
+            'upload_date': '20211225',
+            'duration': 416,
+            'view_count': int,
+            'age_limit': 18,
+            'uploader': 'SEXYhub.com',
+            'categories': list,
+            'tags': list,
+        },
+    }, {
+        'url': 'https://www.playvids.com/es/U3pBrYhsjXM/pc/dane-jones-cute-redhead-with-perfect-tits-with-mini-vamp',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.playvids.com/embed/U3pBrYhsjXM',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.playvids.com/bKmGLe3IwjZ/sv/brazzers-800-phone-sex-madison-ivy-always-on-the-line',
+        'md5': 'e783986e596cafbf46411a174ab42ba6',
+        'info_dict': {
+            'id': '762385',
+            'display_id': 'bKmGLe3IwjZ',
+            'ext': 'mp4',
+            'title': 'Brazzers - 1 800 Phone Sex: Madison Ivy Always On The Line 6',
+            'description': 'md5:bdcd2db2b8ad85831a491d7c8605dcef',
+            'timestamp': 1516958544,
+            'upload_date': '20180126',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 480,
+            'uploader': 'Brazzers',
+            'age_limit': 18,
+            'view_count': int,
+            'age_limit': 18,
+            'categories': list,
+            'tags': list,
+        },
+    }, {
+        'url': 'https://www.playvids.com/v/47iUho33toY',
+        'md5': 'b056b5049d34b648c1e86497cf4febce',
+        'info_dict': {
+            'id': '700621',
+            'display_id': '47iUho33toY',
+            'ext': 'mp4',
+            'title': 'KATEE OWEN STRIPTIASE IN SEXY RED LINGERIE',
+            'description': None,
+            'timestamp': 1507052209,
+            'upload_date': '20171003',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 332,
+            'uploader': 'Cacerenele',
+            'age_limit': 18,
+            'view_count': int,
+            'categories': list,
+            'tags': list,
+        }
+    }, {
+        'url': 'https://www.playvids.com/z3_7iwWCmqt/sexy-teen-filipina-striptease-beautiful-pinay-bargirl-strips-and-dances',
+        'md5': 'efa09be9f031314b7b7e3bc6510cd0df',
+        'info_dict': {
+            'id': '1523518',
+            'display_id': 'z3_7iwWCmqt',
+            'ext': 'mp4',
+            'title': 'SEXY TEEN FILIPINA STRIPTEASE - Beautiful Pinay Bargirl Strips and Dances',
+            'description': None,
+            'timestamp': 1607470323,
+            'upload_date': '20201208',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'duration': 593,
+            'uploader': 'yorours',
+            'age_limit': 18,
+            'view_count': int,
+            'categories': list,
+            'tags': list,
+        },
+    }]
+    _DOMAIN = 'www.playvids.com'
+
+    def _get_detail(self, html):
+        return get_element_by_class('detail-block', html)

From 604762a9f8fa21de3f7349bd612c4f34941a5d20 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 11 Nov 2022 00:49:13 +0000
Subject: [PATCH 568/860] [common:jwplayer] Improve jwplayer extraction and
 parsing (#31000)

* don't crash parser if jwplayer_data is invalid (empty, or no formats)
* use `label` in `sources[n]` as `format_id`
* relax `jwplayer().setup(...)` RE (also rework PR #27274 enhancement)
* detect more manifest formats in _parse_jwplayer_formats() (from PR #29596)
* improve metadata extraction (from PR #25433)
* remember URLs in a set
* use parse_resolution() in format
* extract filesize in format (from yt-dlp)

Co-authored-by: kikuyan <kikuyan@users.noreply.github.com>
Co-authored-by: martin54 <martin54@users.noreply.github.com>
---
 youtube_dl/extractor/common.py | 40 ++++++++++++++++++++++++----------
 1 file changed, 29 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 1f33a1e06..a0a796d7b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -70,6 +70,7 @@ from ..utils import (
     str_or_none,
     str_to_int,
     strip_or_none,
+    try_get,
     unescapeHTML,
     unified_strdate,
     unified_timestamp,
@@ -2713,7 +2714,7 @@ class InfoExtractor(object):
 
     def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
         mobj = re.search(
-            r'(?s)jwplayer\((?P<quote>[\'"])[^\'" ]+(?P=quote)\)(?!</script>).*?\.setup\s*\((?P<options>[^)]+)\)',
+            r'''(?s)jwplayer\s*\(\s*(?P<q>'|")(?!(?P=q)).+(?P=q)\s*\)(?!</script>).*?\.\s*setup\s*\(\s*(?P<options>(?:\([^)]*\)|[^)])+)\s*\)''',
             webpage)
         if mobj:
             try:
@@ -2734,9 +2735,14 @@ class InfoExtractor(object):
 
     def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True,
                              m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
+        flat_pl = try_get(jwplayer_data, lambda x: x.get('playlist') or True)
+        if flat_pl is None:
+            # not even a dict
+            return []
+
         # JWPlayer backward compatibility: flattened playlists
         # https://github.com/jwplayer/jwplayer/blob/v7.4.3/src/js/api/config.js#L81-L96
-        if 'playlist' not in jwplayer_data:
+        if flat_pl is True:
             jwplayer_data = {'playlist': [jwplayer_data]}
 
         entries = []
@@ -2784,6 +2790,13 @@ class InfoExtractor(object):
                 'timestamp': int_or_none(video_data.get('pubdate')),
                 'duration': float_or_none(jwplayer_data.get('duration') or video_data.get('duration')),
                 'subtitles': subtitles,
+                'alt_title': clean_html(video_data.get('subtitle')),  # attributes used e.g. by Tele5 ...
+                'genre': clean_html(video_data.get('genre')),
+                'channel': clean_html(dict_get(video_data, ('category', 'channel'))),
+                'season_number': int_or_none(video_data.get('season')),
+                'episode_number': int_or_none(video_data.get('episode')),
+                'release_year': int_or_none(video_data.get('releasedate')),
+                'age_limit': int_or_none(video_data.get('age_restriction')),
             }
             # https://github.com/jwplayer/jwplayer/blob/master/src/js/utils/validator.js#L32
             if len(formats) == 1 and re.search(r'^(?:http|//).*(?:youtube\.com|youtu\.be)/.+', formats[0]['url']):
@@ -2792,7 +2805,9 @@ class InfoExtractor(object):
                     'url': formats[0]['url'],
                 })
             else:
-                self._sort_formats(formats)
+                # avoid exception in case of only sttls
+                if formats:
+                    self._sort_formats(formats)
                 entry['formats'] = formats
             entries.append(entry)
         if len(entries) == 1:
@@ -2802,7 +2817,7 @@ class InfoExtractor(object):
 
     def _parse_jwplayer_formats(self, jwplayer_sources_data, video_id=None,
                                 m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
-        urls = []
+        urls = set()
         formats = []
         for source in jwplayer_sources_data:
             if not isinstance(source, dict):
@@ -2811,14 +2826,14 @@ class InfoExtractor(object):
                 base_url, self._proto_relative_url(source.get('file')))
             if not source_url or source_url in urls:
                 continue
-            urls.append(source_url)
+            urls.add(source_url)
             source_type = source.get('type') or ''
             ext = mimetype2ext(source_type) or determine_ext(source_url)
-            if source_type == 'hls' or ext == 'm3u8':
+            if source_type == 'hls' or ext == 'm3u8' or 'format=m3u8-aapl' in source_url:
                 formats.extend(self._extract_m3u8_formats(
                     source_url, video_id, 'mp4', entry_protocol='m3u8_native',
                     m3u8_id=m3u8_id, fatal=False))
-            elif source_type == 'dash' or ext == 'mpd':
+            elif source_type == 'dash' or ext == 'mpd' or 'format=mpd-time-csf' in source_url:
                 formats.extend(self._extract_mpd_formats(
                     source_url, video_id, mpd_id=mpd_id, fatal=False))
             elif ext == 'smil':
@@ -2833,20 +2848,23 @@ class InfoExtractor(object):
                     'ext': ext,
                 })
             else:
+                format_id = str_or_none(source.get('label'))
                 height = int_or_none(source.get('height'))
-                if height is None:
+                if height is None and format_id:
                     # Often no height is provided but there is a label in
                     # format like "1080p", "720p SD", or 1080.
-                    height = int_or_none(self._search_regex(
-                        r'^(\d{3,4})[pP]?(?:\b|$)', compat_str(source.get('label') or ''),
-                        'height', default=None))
+                    height = parse_resolution(format_id).get('height')
                 a_format = {
                     'url': source_url,
                     'width': int_or_none(source.get('width')),
                     'height': height,
                     'tbr': int_or_none(source.get('bitrate'), scale=1000),
+                    'filesize': int_or_none(source.get('filesize')),
                     'ext': ext,
                 }
+                if format_id:
+                    a_format['format_id'] = format_id
+
                 if source_url.startswith('rtmp'):
                     a_format['ext'] = 'flv'
                     # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as

From c2f9be3e63a000cf20e9e4ad789a4f5453d00eb7 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 12 Nov 2022 11:55:05 +0000
Subject: [PATCH 569/860] [generic] Add KVS player extraction

---
 youtube_dl/extractor/generic.py | 183 ++++++++++++++++++++++++++++++++
 1 file changed, 183 insertions(+)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index a9c064105..01e406750 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -28,6 +28,7 @@ from ..utils import (
     mimetype2ext,
     orderedSet,
     parse_duration,
+    parse_resolution,
     sanitized_Request,
     smuggle_url,
     unescapeHTML,
@@ -2227,6 +2228,97 @@ class GenericIE(InfoExtractor):
             # Sibnet embed (https://help.sibnet.ru/?sibnet_video_embed)
             'url': 'https://phpbb3.x-tk.ru/bbcode-video-sibnet-t24.html',
             'only_matching': True,
+        }, {
+            # KVS Player
+            'url': 'https://www.kvs-demo.com/videos/105/kelis-4th-of-july/',
+            'info_dict': {
+                'id': '105',
+                'display_id': 'kelis-4th-of-july',
+                'ext': 'mp4',
+                'title': 'Kelis - 4th Of July',
+                'thumbnail': r're:https://(?:www\.)?kvs-demo.com/contents/videos_screenshots/0/105/preview.jpg',
+            },
+        }, {
+            # KVS Player
+            'url': 'https://www.kvs-demo.com/embed/105/',
+            'info_dict': {
+                'id': '105',
+                'display_id': 'kelis-4th-of-july',
+                'ext': 'mp4',
+                'title': 'Kelis - 4th Of July / Embed Player',
+                'thumbnail': r're:https://(?:www\.)?kvs-demo.com/contents/videos_screenshots/0/105/preview.jpg',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        }, {
+            # KVS Player
+            'url': 'https://thisvid.com/videos/fruit-is-healthy/',
+            'md5': 'f83e52f409b9139a7efee58ef926a72e',
+            'info_dict': {
+                'id': '7079579',
+                'display_id': 'fruit-is-healthy',
+                'ext': 'mp4',
+                'title': 'Fruit is healthy - ThisVid.com',
+                'thumbnail': 'https://media.thisvid.com/contents/videos_screenshots/7079000/7079579/preview.jpg',
+            }
+        }, {
+            # KVS Player
+            'url': 'https://thisvid.com/embed/7079579/',
+            'info_dict': {
+                'id': '7079579',
+                'display_id': 'fruit-is-healthy',
+                'ext': 'mp4',
+                'title': 'Fruit is healthy - ThisVid.com',
+                'thumbnail': 'https://media.thisvid.com/contents/videos_screenshots/7079000/7079579/preview.jpg',
+            },
+            'params': {
+                'skip_download': True,
+            },
+        }, {
+            # KVS Player
+            'url': 'https://youix.com/video/leningrad-zoj/',
+            'md5': '94f96ba95706dc3880812b27b7d8a2b8',
+            'info_dict': {
+                'id': '18485',
+                'display_id': 'leningrad-zoj',
+                'ext': 'mp4',
+                'title': 'Клип: Ленинград - ЗОЖ скачать, смотреть онлайн | Youix.com',
+                'thumbnail': r're:https://youix.com/contents/videos_screenshots/18000/18485/preview(?:_480x320_youix_com.mp4)?\.jpg',
+            },
+        }, {
+            # KVS Player
+            'url': 'https://youix.com/embed/18485',
+            'md5': '94f96ba95706dc3880812b27b7d8a2b8',
+            'info_dict': {
+                'id': '18485',
+                'display_id': 'leningrad-zoj',
+                'ext': 'mp4',
+                'title': 'Ленинград - ЗОЖ',
+                'thumbnail': r're:https://youix.com/contents/videos_screenshots/18000/18485/preview(?:_480x320_youix_com.mp4)?\.jpg',
+            },
+        }, {
+            # KVS Player
+            'url': 'https://bogmedia.org/videos/21217/40-nochey-40-nights-2016/',
+            'md5': '94166bdb26b4cb1fb9214319a629fc51',
+            'info_dict': {
+                'id': '21217',
+                'display_id': '40-nochey-2016',
+                'ext': 'mp4',
+                'title': '40 ночей (2016) - BogMedia.org',
+                'thumbnail': 'https://bogmedia.org/contents/videos_screenshots/21000/21217/preview_480p.mp4.jpg',
+            },
+        }, {
+            # KVS Player (for sites that serve kt_player.js via non-https urls)
+            'url': 'http://www.camhub.world/embed/389508',
+            'md5': 'fbe89af4cfb59c8fd9f34a202bb03e32',
+            'info_dict': {
+                'id': '389508',
+                'display_id': 'syren-de-mer-onlyfans-05-07-2020have-a-happy-safe-holiday5f014e68a220979bdb8cd-source',
+                'ext': 'mp4',
+                'title': 'Syren De Mer  onlyfans_05-07-2020Have_a_happy_safe_holiday5f014e68a220979bdb8cd_source / Embed плеер',
+                'thumbnail': r're:https?://www\.camhub\.world/contents/videos_screenshots/389000/389508/preview\.mp4\.jpg',
+            },
         },
     ]
 
@@ -2332,6 +2424,87 @@ class GenericIE(InfoExtractor):
             'title': title,
         }
 
+    def _extract_kvs(self, url, webpage, video_id):
+
+        def getlicensetoken(license):
+            modlicense = license.replace('$', '').replace('0', '1')
+            center = int(len(modlicense) / 2)
+            fronthalf = int(modlicense[:center + 1])
+            backhalf = int(modlicense[center:])
+
+            modlicense = compat_str(4 * abs(fronthalf - backhalf))
+
+            def parts():
+                for o in range(0, center + 1):
+                    for i in range(1, 5):
+                        yield compat_str((int(license[o + i]) + int(modlicense[o])) % 10)
+
+            return ''.join(parts())
+
+        def getrealurl(video_url, license_code):
+            if not video_url.startswith('function/0/'):
+                return video_url  # not obfuscated
+
+            url_path, _, url_query = video_url.partition('?')
+            urlparts = url_path.split('/')[2:]
+            license = getlicensetoken(license_code)
+            newmagic = urlparts[5][:32]
+
+            def spells(x, o):
+                l = (o + sum(int(n) for n in license[o:])) % 32
+                for i in range(0, len(x)):
+                    yield {l: x[o], o: x[l]}.get(i, x[i])
+
+            for o in range(len(newmagic) - 1, -1, -1):
+                newmagic = ''.join(spells(newmagic, o))
+
+            urlparts[5] = newmagic + urlparts[5][32:]
+            return '/'.join(urlparts) + '?' + url_query
+
+        flashvars = self._search_regex(
+            r'(?s)<script\b[^>]*>.*?var\s+flashvars\s*=\s*(\{.+?\});.*?</script>',
+            webpage, 'flashvars')
+        flashvars = self._parse_json(flashvars, video_id, transform_source=js_to_json)
+
+        # extract the part after the last / as the display_id from the
+        # canonical URL.
+        display_id = self._search_regex(
+            r'(?:<link href="https?://[^"]+/(.+?)/?" rel="canonical"\s*/?>'
+            r'|<link rel="canonical" href="https?://[^"]+/(.+?)/?"\s*/?>)',
+            webpage, 'display_id', fatal=False
+        )
+        title = self._html_search_regex(r'<(?:h1|title)>(?:Video: )?(.+?)</(?:h1|title)>', webpage, 'title')
+
+        thumbnail = flashvars['preview_url']
+        if thumbnail.startswith('//'):
+            protocol, _, _ = url.partition('/')
+            thumbnail = protocol + thumbnail
+
+        url_keys = list(filter(re.compile(r'^video_(?:url|alt_url\d*)$').match, flashvars.keys()))
+        formats = []
+        for key in url_keys:
+            if '/get_file/' not in flashvars[key]:
+                continue
+            format_id = flashvars.get(key + '_text', key)
+            formats.append(merge_dicts(
+                parse_resolution(format_id) or parse_resolution(flashvars[key]), {
+                    'url': getrealurl(flashvars[key], flashvars['license_code']),
+                    'format_id': format_id,
+                    'ext': 'mp4',
+                }))
+            if not formats[-1].get('height'):
+                formats[-1]['quality'] = 1
+
+        self._sort_formats(formats)
+
+        return {
+            'id': flashvars['video_id'],
+            'display_id': display_id,
+            'title': title,
+            'thumbnail': thumbnail,
+            'formats': formats,
+        }
+
     def _real_extract(self, url):
         if url.startswith('//'):
             return self.url_result(self.http_scheme() + url)
@@ -3389,6 +3562,16 @@ class GenericIE(InfoExtractor):
                 info_dict['formats'] = formats
                 return info_dict
 
+        # Look for generic KVS player (before ld+json for tests)
+        found = re.search(
+            r'<script\b[^>]+?\bsrc\s*=\s*(["\'])https?://(?:\S+?/)+kt_player\.js\?v=(?P<ver>(?P<maj_ver>\d+)(\.\d+)+)\1[^>]*>',
+            webpage)
+        if found:
+            self.report_extraction('KVS Player')
+            if found.group('maj_ver') not in ('4', '5', '6'):
+                self.report_warning('Untested major version (%s) in player engine - download may fail.' % (found.group('ver'), ))
+            return self._extract_kvs(url, webpage, video_id)
+
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
             webpage, video_id, default={}, expected_type='VideoObject')

From 1a4fbe8462f5e531a891aeac7db6c0bde49c5536 Mon Sep 17 00:00:00 2001
From: FraFraFra-LongD <85188920+FraFraFra-LongD@users.noreply.github.com>
Date: Sun, 13 Nov 2022 14:22:04 +0100
Subject: [PATCH 570/860] Added ThisVid.com support (#29187)

* add ThisVidIE, ThisVidMemberIE, ThisVidPlaylistIE
* redirect embed to main page for more metadata
* use KVS extraction newly added to GenericIE and remove duplicate tests
* also add MrDeepFake etc compat to GenericIE
(closes #22390)

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/extractors.py |   5 +
 youtube_dl/extractor/generic.py    |  54 ++++---
 youtube_dl/extractor/thisvid.py    | 218 +++++++++++++++++++++++++++++
 3 files changed, 249 insertions(+), 28 deletions(-)
 create mode 100644 youtube_dl/extractor/thisvid.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 4d9f37424..947cbe8fd 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1265,6 +1265,11 @@ from .theweatherchannel import TheWeatherChannelIE
 from .thisamericanlife import ThisAmericanLifeIE
 from .thisav import ThisAVIE
 from .thisoldhouse import ThisOldHouseIE
+from .thisvid import (
+    ThisVidIE,
+    ThisVidMemberIE,
+    ThisVidPlaylistIE,
+)
 from .threeqsdn import ThreeQSDNIE
 from .tiktok import (
     TikTokIE,
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 01e406750..597611157 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2252,31 +2252,7 @@ class GenericIE(InfoExtractor):
                 'skip_download': True,
             },
         }, {
-            # KVS Player
-            'url': 'https://thisvid.com/videos/fruit-is-healthy/',
-            'md5': 'f83e52f409b9139a7efee58ef926a72e',
-            'info_dict': {
-                'id': '7079579',
-                'display_id': 'fruit-is-healthy',
-                'ext': 'mp4',
-                'title': 'Fruit is healthy - ThisVid.com',
-                'thumbnail': 'https://media.thisvid.com/contents/videos_screenshots/7079000/7079579/preview.jpg',
-            }
-        }, {
-            # KVS Player
-            'url': 'https://thisvid.com/embed/7079579/',
-            'info_dict': {
-                'id': '7079579',
-                'display_id': 'fruit-is-healthy',
-                'ext': 'mp4',
-                'title': 'Fruit is healthy - ThisVid.com',
-                'thumbnail': 'https://media.thisvid.com/contents/videos_screenshots/7079000/7079579/preview.jpg',
-            },
-            'params': {
-                'skip_download': True,
-            },
-        }, {
-            # KVS Player
+            # KVS Player (tested also in thisvid.py)
             'url': 'https://youix.com/video/leningrad-zoj/',
             'md5': '94f96ba95706dc3880812b27b7d8a2b8',
             'info_dict': {
@@ -2306,6 +2282,7 @@ class GenericIE(InfoExtractor):
                 'display_id': '40-nochey-2016',
                 'ext': 'mp4',
                 'title': '40 ночей (2016) - BogMedia.org',
+                'description': 'md5:4e6d7d622636eb7948275432eb256dc3',
                 'thumbnail': 'https://bogmedia.org/contents/videos_screenshots/21000/21217/preview_480p.mp4.jpg',
             },
         }, {
@@ -2319,6 +2296,18 @@ class GenericIE(InfoExtractor):
                 'title': 'Syren De Mer  onlyfans_05-07-2020Have_a_happy_safe_holiday5f014e68a220979bdb8cd_source / Embed плеер',
                 'thumbnail': r're:https?://www\.camhub\.world/contents/videos_screenshots/389000/389508/preview\.mp4\.jpg',
             },
+        }, {
+            'url': 'https://mrdeepfakes.com/video/5/selena-gomez-pov-deep-fakes',
+            'md5': 'fec4ad5ec150f655e0c74c696a4a2ff4',
+            'info_dict': {
+                'id': '5',
+                'display_id': 'selena-gomez-pov-deep-fakes',
+                'ext': 'mp4',
+                'title': 'Selena Gomez POV (Deep Fakes) DeepFake Porn - MrDeepFakes',
+                'description': 'md5:17d1f84b578c9c26875ac5ef9a932354',
+                'height': 720,
+                'age_limit': 18,
+            },
         },
     ]
 
@@ -2491,6 +2480,7 @@ class GenericIE(InfoExtractor):
                     'url': getrealurl(flashvars[key], flashvars['license_code']),
                     'format_id': format_id,
                     'ext': 'mp4',
+                    'http_headers': {'Referer': url},
                 }))
             if not formats[-1].get('height'):
                 formats[-1]['quality'] = 1
@@ -2713,9 +2703,15 @@ class GenericIE(InfoExtractor):
         # but actually don't.
         AGE_LIMIT_MARKERS = [
             r'Proudly Labeled <a href="http://www\.rtalabel\.org/" title="Restricted to Adults">RTA</a>',
+            r'>[^<]*you acknowledge you are at least (\d+) years old',
         ]
-        if any(re.search(marker, webpage) for marker in AGE_LIMIT_MARKERS):
-            age_limit = 18
+        for marker in AGE_LIMIT_MARKERS:
+            m = re.search(marker, webpage)
+            if not m:
+                continue
+            age_limit = max(
+                age_limit or 0,
+                int_or_none(m.groups() and m.group(1), default=18))
 
         # video uploader is domain name
         video_uploader = self._search_regex(
@@ -3570,7 +3566,9 @@ class GenericIE(InfoExtractor):
             self.report_extraction('KVS Player')
             if found.group('maj_ver') not in ('4', '5', '6'):
                 self.report_warning('Untested major version (%s) in player engine - download may fail.' % (found.group('ver'), ))
-            return self._extract_kvs(url, webpage, video_id)
+            return merge_dicts(
+                self._extract_kvs(url, webpage, video_id),
+                info_dict)
 
         # Looking for http://schema.org/VideoObject
         json_ld = self._search_json_ld(
diff --git a/youtube_dl/extractor/thisvid.py b/youtube_dl/extractor/thisvid.py
new file mode 100644
index 000000000..bc4bcb2d1
--- /dev/null
+++ b/youtube_dl/extractor/thisvid.py
@@ -0,0 +1,218 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+import itertools
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_urlparse,
+)
+from ..utils import (
+    clean_html,
+    get_element_by_class,
+    int_or_none,
+    merge_dicts,
+    url_or_none,
+    urljoin,
+)
+
+
+class ThisVidIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?thisvid\.com/(?P<type>videos|embed)/(?P<id>[A-Za-z0-9-]+)'
+    _TESTS = [{
+        'url': 'https://thisvid.com/videos/sitting-on-ball-tight-jeans/',
+        'md5': '839becb572995687e11a69dc4358a386',
+        'info_dict': {
+            'id': '3533241',
+            'ext': 'mp4',
+            'title': 'Sitting on ball tight jeans',
+            'description': 'md5:372353bb995883d1b65fddf507489acd',
+            'thumbnail': r're:https?://\w+\.thisvid\.com/(?:[^/]+/)+3533241/preview\.jpg',
+            'uploader_id': '150629',
+            'uploader': 'jeanslevisjeans',
+            'age_limit': 18,
+        }
+    }, {
+        'url': 'https://thisvid.com/embed/3533241/',
+        'md5': '839becb572995687e11a69dc4358a386',
+        'info_dict': {
+            'id': '3533241',
+            'ext': 'mp4',
+            'title': 'Sitting on ball tight jeans',
+            'thumbnail': r're:https?://\w+\.thisvid\.com/(?:[^/]+/)+3533241/preview\.jpg',
+            'uploader_id': '150629',
+            'uploader': 'jeanslevisjeans',
+            'age_limit': 18,
+        }
+    }]
+
+    def _real_extract(self, url):
+        main_id, type_ = re.match(self._VALID_URL, url).group('id', 'type')
+        webpage = self._download_webpage(url, main_id)
+
+        title = self._html_search_regex(
+            r'<title\b[^>]*?>(?:Video:\s+)?(.+?)(?:\s+-\s+ThisVid(?:\.com| tube))?</title>',
+            webpage, 'title')
+
+        if type_ == 'embed':
+            # look for more metadata
+            video_alt_url = url_or_none(self._search_regex(
+                r'''video_alt_url\s*:\s+'(%s/)',''' % (self._VALID_URL, ),
+                webpage, 'video_alt_url', default=None))
+            if video_alt_url and video_alt_url != url:
+                webpage = self._download_webpage(
+                    video_alt_url, main_id,
+                    note='Redirecting embed to main page', fatal=False) or webpage
+
+        video_holder = get_element_by_class('video-holder', webpage) or ''
+        if '>This video is a private video' in video_holder:
+            self.raise_login_required(
+                (clean_html(video_holder) or 'Private video').split('\n', 1)[0])
+
+        uploader = self._html_search_regex(
+            r'''(?s)<span\b[^>]*>Added by:\s*</span><a\b[^>]+\bclass\s*=\s*["']author\b[^>]+\bhref\s*=\s*["']https://thisvid\.com/members/([0-9]+/.{3,}?)\s*</a>''',
+            webpage, 'uploader', default='')
+        uploader = re.split(r'''/["'][^>]*>\s*''', uploader)
+        if len(uploader) == 2:
+            # id must be non-empty, uploader could be ''
+            uploader_id, uploader = uploader
+            uploader = uploader or None
+        else:
+            uploader_id = uploader = None
+
+        return merge_dicts({
+            '_type': 'url_transparent',
+            'title': title,
+            'age_limit': 18,
+            'uploader': uploader,
+            'uploader_id': uploader_id,
+        }, self.url_result(url, ie='Generic'))
+
+
+class ThisVidMemberIE(InfoExtractor):
+    _VALID_URL = r'https?://thisvid\.com/members/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://thisvid.com/members/2140501/',
+        'info_dict': {
+            'id': '2140501',
+            'title': 'Rafflesia\'s Profile',
+        },
+        'playlist_mincount': 16,
+    }, {
+        'url': 'https://thisvid.com/members/2140501/favourite_videos/',
+        'info_dict': {
+            'id': '2140501',
+            'title': 'Rafflesia\'s Favourite Videos',
+        },
+        'playlist_mincount': 15,
+    }, {
+        'url': 'https://thisvid.com/members/636468/public_videos/',
+        'info_dict': {
+            'id': '636468',
+            'title': 'Happymouth\'s Public Videos',
+        },
+        'playlist_mincount': 196,
+    },
+    ]
+
+    def _urls(self, html):
+        for m in re.finditer(r'''<a\b[^>]+\bhref\s*=\s*["'](?P<url>%s\b)[^>]+>''' % (ThisVidIE._VALID_URL, ), html):
+            yield m.group('url')
+
+    def _real_extract(self, url):
+        pl_id = self._match_id(url)
+        webpage = self._download_webpage(url, pl_id)
+
+        title = re.split(
+            r'(?i)\s*\|\s*ThisVid\.com\s*$',
+            self._og_search_title(webpage, default=None) or self._html_search_regex(r'(?s)<title\b[^>]*>(.+?)</title', webpage, 'title', fatal=False) or '', 1)[0] or None
+
+        def entries(page_url, html=None):
+            for page in itertools.count(1):
+                if not html:
+                    html = self._download_webpage(
+                        page_url, pl_id, note='Downloading page %d' % (page, ),
+                        fatal=False) or ''
+                for u in self._urls(html):
+                    yield u
+                next_page = get_element_by_class('pagination-next', html) or ''
+                if next_page:
+                    # member list page
+                    next_page = urljoin(url, self._search_regex(
+                        r'''<a\b[^>]+\bhref\s*=\s*("|')(?P<url>(?!#)(?:(?!\1).)+)''',
+                        next_page, 'next page link', group='url', default=None))
+                # in case a member page should have pagination-next with empty link, not just `else:`
+                if next_page is None:
+                    # playlist page
+                    parsed_url = compat_urlparse.urlparse(page_url)
+                    base_path, num = parsed_url.path.rsplit('/', 1)
+                    num = int_or_none(num)
+                    if num is None:
+                        base_path, num = parsed_url.path.rstrip('/'), 1
+                    parsed_url = parsed_url._replace(path=base_path + ('/%d' % (num + 1, )))
+                    next_page = compat_urlparse.urlunparse(parsed_url)
+                    if page_url == next_page:
+                        next_page = None
+                if not next_page:
+                    break
+                page_url, html = next_page, None
+
+        return self.playlist_from_matches(
+            entries(url, webpage), playlist_id=pl_id, playlist_title=title, ie='ThisVid')
+
+
+class ThisVidPlaylistIE(ThisVidMemberIE):
+    _VALID_URL = r'https?://thisvid\.com/playlist/(?P<id>\d+)/video/(?P<video_id>[A-Za-z0-9-]+)'
+    _TESTS = [{
+        'url': 'https://thisvid.com/playlist/6615/video/big-italian-booty-28/',
+        'info_dict': {
+            'id': '6615',
+            'title': 'Underwear Stuff',
+        },
+        'playlist_mincount': 200,
+    }, {
+        'url': 'https://thisvid.com/playlist/6615/video/big-italian-booty-28/',
+        'info_dict': {
+            'id': '1072387',
+            'ext': 'mp4',
+            'title': 'Big Italian Booty 28',
+            'description': 'md5:1bccf7b13765e18fb27bf764dba7ede2',
+            'uploader_id': '367912',
+            'uploader': 'Jcmusclefun',
+            'age_limit': 18,
+        },
+        'params': {
+            'noplaylist': True,
+        },
+    }]
+
+    def _get_video_url(self, pl_url):
+        video_id = re.match(self._VALID_URL, pl_url).group('video_id')
+        return urljoin(pl_url, '/videos/%s/' % (video_id, ))
+
+    def _urls(self, html):
+        for m in re.finditer(r'''<a\b[^>]+\bhref\s*=\s*["'](?P<url>%s\b)[^>]+>''' % (self._VALID_URL, ), html):
+            yield self._get_video_url(m.group('url'))
+
+    def _real_extract(self, url):
+        pl_id = self._match_id(url)
+
+        if self._downloader.params.get('noplaylist'):
+            self.to_screen('Downloading just the featured video because of --no-playlist')
+            return self.url_result(self._get_video_url(url), 'ThisVid')
+
+        self.to_screen(
+            'Downloading playlist %s - add --no-playlist to download just the featured video' % (pl_id, ))
+        result = super(ThisVidPlaylistIE, self)._real_extract(url)
+
+        # rework title returned as `the title - the title`
+        title = result['title']
+        t_len = len(title)
+        if t_len > 5 and t_len % 2 != 0:
+            t_len = t_len // 2
+            if title[t_len] == '-':
+                title = [t.strip() for t in (title[:t_len], title[t_len + 1:])]
+                if title[0] and title[0] == title[1]:
+                    result['title'] = title[0]
+        return result

From fc2beab0e701c497a003f11fef5c0df54fba1da3 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 13 Nov 2022 14:59:30 +0000
Subject: [PATCH 571/860] [generic] Improve KVS (etc) extraction

* detect kt_player('kt_player', 'https://.../kt_player.swf?v=5...
* detect age limit if 18 USC 2257 is mentioned
* test with shooshtime.com

Partially resolves #31332.
---
 youtube_dl/extractor/generic.py | 29 ++++++++++++++++++++++-------
 1 file changed, 22 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 597611157..3e8281ed3 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -36,6 +36,7 @@ from ..utils import (
     unsmuggle_url,
     UnsupportedError,
     url_or_none,
+    urljoin,
     xpath_attr,
     xpath_text,
     xpath_with_ns,
@@ -2308,6 +2309,17 @@ class GenericIE(InfoExtractor):
                 'height': 720,
                 'age_limit': 18,
             },
+        }, {
+            'url': 'https://shooshtime.com/videos/284002/just-out-of-the-shower-joi/',
+            'md5': 'e2f0a4c329f7986280b7328e24036d60',
+            'info_dict': {
+                'id': '284002',
+                'display_id': 'just-out-of-the-shower-joi',
+                'ext': 'mp4',
+                'title': 'Just Out Of The Shower JOI - Shooshtime',
+                'height': 720,
+                'age_limit': 18,
+            },
         },
     ]
 
@@ -2477,7 +2489,7 @@ class GenericIE(InfoExtractor):
             format_id = flashvars.get(key + '_text', key)
             formats.append(merge_dicts(
                 parse_resolution(format_id) or parse_resolution(flashvars[key]), {
-                    'url': getrealurl(flashvars[key], flashvars['license_code']),
+                    'url': urljoin(url, getrealurl(flashvars[key], flashvars['license_code'])),
                     'format_id': format_id,
                     'ext': 'mp4',
                     'http_headers': {'Referer': url},
@@ -2704,6 +2716,7 @@ class GenericIE(InfoExtractor):
         AGE_LIMIT_MARKERS = [
             r'Proudly Labeled <a href="http://www\.rtalabel\.org/" title="Restricted to Adults">RTA</a>',
             r'>[^<]*you acknowledge you are at least (\d+) years old',
+            r'>\s*(?:18\s+U(?:\.S\.C\.|SC)\s+)?(?:§+\s*)?2257\b',
         ]
         for marker in AGE_LIMIT_MARKERS:
             m = re.search(marker, webpage)
@@ -3559,13 +3572,15 @@ class GenericIE(InfoExtractor):
                 return info_dict
 
         # Look for generic KVS player (before ld+json for tests)
-        found = re.search(
-            r'<script\b[^>]+?\bsrc\s*=\s*(["\'])https?://(?:\S+?/)+kt_player\.js\?v=(?P<ver>(?P<maj_ver>\d+)(\.\d+)+)\1[^>]*>',
-            webpage)
+        found = self._search_regex(
+            (r'<script\b[^>]+?\bsrc\s*=\s*(["\'])https?://(?:\S+?/)+kt_player\.js\?v=(?P<ver>\d+(?:\.\d+)+)\1[^>]*>',
+             # kt_player('kt_player', 'https://i.shoosh.co/player/kt_player.swf?v=5.5.1', ...
+             r'kt_player\s*\(\s*(["\'])(?:(?!\1)[\w\W])+\1\s*,\s*(["\'])https?://(?:\S+?/)+kt_player\.swf\?v=(?P<ver>\d+(?:\.\d+)+)\2\s*,', 
+            ), webpage, 'KVS player', group='ver', default=False)
         if found:
-            self.report_extraction('KVS Player')
-            if found.group('maj_ver') not in ('4', '5', '6'):
-                self.report_warning('Untested major version (%s) in player engine - download may fail.' % (found.group('ver'), ))
+            self.report_extraction('%s: KVS Player' % (video_id, ))
+            if found.split('.')[0] not in ('4', '5', '6'):
+                self.report_warning('Untested major version (%s) in player engine - download may fail.' % (found, ))
             return merge_dicts(
                 self._extract_kvs(url, webpage, video_id),
                 info_dict)

From 195f22f679330549882a8234e7234942893a4902 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 13 Nov 2022 15:09:29 +0000
Subject: [PATCH 572/860] [generic] Improve KVS (etc) extraction

---
 youtube_dl/extractor/generic.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 3e8281ed3..0e473e952 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -3575,8 +3575,8 @@ class GenericIE(InfoExtractor):
         found = self._search_regex(
             (r'<script\b[^>]+?\bsrc\s*=\s*(["\'])https?://(?:\S+?/)+kt_player\.js\?v=(?P<ver>\d+(?:\.\d+)+)\1[^>]*>',
              # kt_player('kt_player', 'https://i.shoosh.co/player/kt_player.swf?v=5.5.1', ...
-             r'kt_player\s*\(\s*(["\'])(?:(?!\1)[\w\W])+\1\s*,\s*(["\'])https?://(?:\S+?/)+kt_player\.swf\?v=(?P<ver>\d+(?:\.\d+)+)\2\s*,', 
-            ), webpage, 'KVS player', group='ver', default=False)
+             r'kt_player\s*\(\s*(["\'])(?:(?!\1)[\w\W])+\1\s*,\s*(["\'])https?://(?:\S+?/)+kt_player\.swf\?v=(?P<ver>\d+(?:\.\d+)+)\2\s*,',
+             ), webpage, 'KVS player', group='ver', default=False)
         if found:
             self.report_extraction('%s: KVS Player' % (video_id, ))
             if found.split('.')[0] not in ('4', '5', '6'):

From 14ef89a8dab4f6ba6185d6f5bf0317a705d7b842 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Wed, 1 Feb 2023 09:39:49 +0530
Subject: [PATCH 573/860] Support `if` statements

Fix for yt-dlp/yt_dlp#6131
Closes #31509
---
 test/test_jsinterp.py          | 32 ++++++++++++++++++++++++++++++++
 test/test_youtube_signature.py |  4 ++++
 youtube_dl/jsinterp.py         | 21 ++++++++++++++++++---
 3 files changed, 54 insertions(+), 3 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 5121c8cf8..c47def737 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -158,6 +158,38 @@ class TestJSInterpreter(unittest.TestCase):
         self.assertEqual(jsi.call_function('z'), 5)
         self.assertEqual(jsi.call_function('y'), 2)
 
+    def test_if(self):
+        jsi = JSInterpreter('''
+        function x() {
+            let a = 9;
+            if (0==0) {a++}
+            return a
+        }''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+        jsi = JSInterpreter('''
+        function x() {
+            if (0==0) {return 10}
+        }''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+        jsi = JSInterpreter('''
+        function x() {
+            if (0!=0) {return 1}
+            else {return 10}
+        }''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
+        """  # Unsupported
+        jsi = JSInterpreter('''
+        function x() {
+            if (0!=0) {return 1}
+            else if (1==0) {return 2}
+            else {return 10}
+        }''')
+        self.assertEqual(jsi.call_function('x'), 10)
+        """
+
     def test_for_loop(self):
         # function x() { a=0; for (i=0; i-10; i++) {a++} a }
         jsi = JSInterpreter('''
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 4e678cae0..ac37ffa45 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -135,6 +135,10 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/5a3b6271/player_ias.vflset/en_US/base.js',
         'B2j7f_UPT4rfje85Lu_e', 'm5DmNymaGQ5RdQ',
     ),
+    (
+        'https://www.youtube.com/s/player/dac945fd/player_ias.vflset/en_US/base.js',
+        'o8BkRxXhuYsBCWi6RplPdP', '3Lx32v_hmzTm6A',
+    ),
 ]
 
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 530a705b4..9a3b8d7f2 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -214,7 +214,7 @@ class JSInterpreter(object):
         def __init__(self, msg, *args, **kwargs):
             expr = kwargs.pop('expr', None)
             if expr is not None:
-                msg = '{0} in: {1!r:.100}'.format(msg.rstrip(), expr)
+                msg = '{0} in: {1!r}'.format(msg.rstrip(), expr[:100])
             super(JSInterpreter.Exception, self).__init__(msg, *args, **kwargs)
 
     @classmethod
@@ -268,7 +268,7 @@ class JSInterpreter(object):
                 elif in_quote == '/' and char in '[]':
                     in_regex_char_group = char == '['
             escaping = not escaping and in_quote and char == '\\'
-            after_op = not in_quote and (char in cls.OP_CHARS or char == '[' or (char.isspace() and after_op))
+            after_op = not in_quote and (char in cls.OP_CHARS or (char.isspace() and after_op))
 
             if char != delim[pos] or any(counters.values()) or in_quote:
                 pos = skipping = 0
@@ -301,7 +301,7 @@ class JSInterpreter(object):
         separated = list(cls._separate(expr, delim, 1))
 
         if len(separated) < 2:
-            raise cls.Exception('No terminating paren {delim} in {expr:.100}'.format(**locals()))
+            raise cls.Exception('No terminating paren {delim} in {expr}'.format(**locals()))
         return separated[0][1:].strip(), separated[1].strip()
 
     @staticmethod
@@ -428,10 +428,25 @@ class JSInterpreter(object):
 
         m = re.match(r'''(?x)
                 (?P<try>try)\s*\{|
+                (?P<if>if)\s*\(|
                 (?P<switch>switch)\s*\(|
                 (?P<for>for)\s*\(
                 ''', expr)
         md = m.groupdict() if m else {}
+        if md.get('if'):
+            cndn, expr = self._separate_at_paren(expr[m.end() - 1:])
+            if_expr, expr = self._separate_at_paren(expr.lstrip())
+            # TODO: "else if" is not handled
+            else_expr = None
+            m = re.match(r'else\s*{', expr)
+            if m:
+                else_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
+            cndn = _js_ternary(self.interpret_expression(cndn, local_vars, allow_recursion))
+            ret, should_abort = self.interpret_statement(
+                if_expr if cndn else else_expr, local_vars, allow_recursion)
+            if should_abort:
+                return ret, True
+
         if md.get('try'):
             try_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
             err = None

From 295736c9cba714fb5de7d1c3dd31d86e50091cf8 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 2 Feb 2023 14:28:32 +0000
Subject: [PATCH 574/860] [jsinterp] Improve parsing * support subset `... else
 if ...` * support `while` * add `RegExp` class * generalise `new` support *
 limited more debug strings * matching test changes

---
 test/test_jsinterp.py  |  53 +++++++++++++-
 youtube_dl/jsinterp.py | 156 +++++++++++++++++++++++++++--------------
 2 files changed, 154 insertions(+), 55 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index c47def737..b5962356c 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -11,8 +11,6 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 import math
 import re
 
-from youtube_dl.compat import compat_re_Pattern
-
 from youtube_dl.jsinterp import JS_Undefined, JSInterpreter
 
 
@@ -140,15 +138,23 @@ class TestJSInterpreter(unittest.TestCase):
         ''')
         self.assertTrue(math.isnan(jsi.call_function('x')))
 
+    def test_Date(self):
         jsi = JSInterpreter('''
         function x() { return new Date('Wednesday 31 December 1969 18:01:26 MDT') - 0; }
         ''')
         self.assertEqual(jsi.call_function('x'), 86000)
+
         jsi = JSInterpreter('''
         function x(dt) { return new Date(dt) - 0; }
         ''')
         self.assertEqual(jsi.call_function('x', 'Wednesday 31 December 1969 18:01:26 MDT'), 86000)
 
+        # date format m/d/y
+        jsi = JSInterpreter('''
+        function x() { return new Date('12/31/1969 18:01:26 MDT') - 0; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 86000)
+
     def test_call(self):
         jsi = JSInterpreter('''
         function x() { return 2; }
@@ -181,6 +187,15 @@ class TestJSInterpreter(unittest.TestCase):
         self.assertEqual(jsi.call_function('x'), 10)
 
         """  # Unsupported
+        jsi = JSInterpreter('''
+        function x() {
+            if (0!=0) return 1;
+            else {return 10}
+        }''')
+        self.assertEqual(jsi.call_function('x'), 10)
+        """
+
+    def test_elseif(self):
         jsi = JSInterpreter('''
         function x() {
             if (0!=0) {return 1}
@@ -188,6 +203,16 @@ class TestJSInterpreter(unittest.TestCase):
             else {return 10}
         }''')
         self.assertEqual(jsi.call_function('x'), 10)
+
+        """  # Unsupported
+        jsi = JSInterpreter('''
+        function x() {
+            if (0!=0) return 1;
+            else if (1==0) {return 2}
+            else {return 10}
+        }''')
+        self.assertEqual(jsi.call_function('x'), 10)
+        # etc
         """
 
     def test_for_loop(self):
@@ -197,6 +222,13 @@ class TestJSInterpreter(unittest.TestCase):
         ''')
         self.assertEqual(jsi.call_function('x'), 10)
 
+    def test_while_loop(self):
+        # function x() { a=0; while (a<10) {a++} a }
+        jsi = JSInterpreter('''
+        function x() { a=0; while (a<10) {a++} return a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 10)
+
     def test_switch(self):
         jsi = JSInterpreter('''
         function x(f) { switch(f){
@@ -415,13 +447,28 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('''
         function x() { let a=/,,[/,913,/](,)}/; return a; }
         ''')
-        self.assertIsInstance(jsi.call_function('x'), compat_re_Pattern)
+        attrs = set(('findall', 'finditer', 'flags', 'groupindex',
+                     'groups', 'match', 'pattern', 'scanner',
+                     'search', 'split', 'sub', 'subn'))
+        self.assertTrue(set(dir(jsi.call_function('x'))) > attrs)
 
         jsi = JSInterpreter('''
         function x() { let a=/,,[/,913,/](,)}/i; return a; }
         ''')
         self.assertEqual(jsi.call_function('x').flags & ~re.U, re.I)
 
+        jsi = JSInterpreter(r'''
+        function x() { let a=[/[)\\]/]; return a[0]; }
+        ''')
+        self.assertEqual(jsi.call_function('x').pattern, r'[)\\]')
+
+        """  # fails
+        jsi = JSInterpreter(r'''
+        function x() { let a=100; a/=/[0-9]+/.exec('divide by 20 today')[0]; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 5)
+        """
+
     def test_char_code_at(self):
         jsi = JSInterpreter('function x(i){return "test".charCodeAt(i)}')
         self.assertEqual(jsi.call_function('x', 0), 116)
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 9a3b8d7f2..1e7b342ac 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -187,19 +187,6 @@ class LocalNameSpace(ChainMap):
 class JSInterpreter(object):
     __named_object_counter = 0
 
-    _RE_FLAGS = {
-        # special knowledge: Python's re flags are bitmask values, current max 128
-        # invent new bitmask values well above that for literal parsing
-        # TODO: new pattern class to execute matches with these flags
-        'd': 1024,  # Generate indices for substring matches
-        'g': 2048,  # Global search
-        'i': re.I,  # Case-insensitive search
-        'm': re.M,  # Multi-line search
-        's': re.S,  # Allows . to match newline characters
-        'u': re.U,  # Treat a pattern as a sequence of unicode code points
-        'y': 4096,  # Perform a "sticky" search that matches starting at the current position in the target string
-    }
-
     _OBJ_NAME = '__youtube_dl_jsinterp_obj'
 
     OP_CHARS = None
@@ -217,9 +204,48 @@ class JSInterpreter(object):
                 msg = '{0} in: {1!r}'.format(msg.rstrip(), expr[:100])
             super(JSInterpreter.Exception, self).__init__(msg, *args, **kwargs)
 
+    class JS_RegExp(object):
+        _RE_FLAGS = {
+            # special knowledge: Python's re flags are bitmask values, current max 128
+            # invent new bitmask values well above that for literal parsing
+            # TODO: new pattern class to execute matches with these flags
+            'd': 1024,  # Generate indices for substring matches
+            'g': 2048,  # Global search
+            'i': re.I,  # Case-insensitive search
+            'm': re.M,  # Multi-line search
+            's': re.S,  # Allows . to match newline characters
+            'u': re.U,  # Treat a pattern as a sequence of unicode code points
+            'y': 4096,  # Perform a "sticky" search that matches starting at the current position in the target string
+        }
+
+        def __init__(self, pattern_txt, flags=''):
+            if isinstance(flags, compat_str):
+                flags, _ = self.regex_flags(flags)
+            # Thx: https://stackoverflow.com/questions/44773522/setattr-on-python2-sre-sre-pattern
+            # First, avoid https://github.com/python/cpython/issues/74534
+            self.__self = re.compile(pattern_txt.replace('[[', r'[\['), flags)
+            for name in dir(self.__self):
+                # Only these? Obviously __class__, __init__.
+                # PyPy creates a __weakref__ attribute with value None
+                # that can't be setattr'd but also can't need to be copied.
+                if name in ('__class__', '__init__', '__weakref__'):
+                    continue
+                setattr(self, name, getattr(self.__self, name))
+
+        @classmethod
+        def regex_flags(cls, expr):
+            flags = 0
+            if not expr:
+                return flags, expr
+            for idx, ch in enumerate(expr):
+                if ch not in cls._RE_FLAGS:
+                    break
+                flags |= cls._RE_FLAGS[ch]
+            return flags, expr[idx + 1:]
+
     @classmethod
     def __op_chars(cls):
-        op_chars = set(';,')
+        op_chars = set(';,[')
         for op in cls._all_operators():
             for c in op[0]:
                 op_chars.add(c)
@@ -231,17 +257,6 @@ class JSInterpreter(object):
         namespace[name] = obj
         return name
 
-    @classmethod
-    def _regex_flags(cls, expr):
-        flags = 0
-        if not expr:
-            return flags, expr
-        for idx, ch in enumerate(expr):
-            if ch not in cls._RE_FLAGS:
-                break
-            flags |= cls._RE_FLAGS[ch]
-        return flags, expr[idx + 1:]
-
     @classmethod
     def _separate(cls, expr, delim=',', max_split=None, skip_delims=None):
         if not expr:
@@ -328,7 +343,7 @@ class JSInterpreter(object):
         try:
             return opfunc(left_val, right_val)
         except Exception as e:
-            raise self.Exception('Failed to evaluate {left_val!r} {op} {right_val!r}'.format(**locals()), expr, cause=e)
+            raise self.Exception('Failed to evaluate {left_val!r:.50} {op} {right_val!r:.50}'.format(**locals()), expr, cause=e)
 
     def _index(self, obj, idx, allow_undefined=False):
         if idx == 'length':
@@ -338,7 +353,7 @@ class JSInterpreter(object):
         except Exception as e:
             if allow_undefined:
                 return JS_Undefined
-            raise self.Exception('Cannot get index {idx}'.format(**locals()), expr=repr(obj), cause=e)
+            raise self.Exception('Cannot get index {idx:.100}'.format(**locals()), expr=repr(obj), cause=e)
 
     def _dump(self, obj, namespace):
         try:
@@ -352,6 +367,7 @@ class JSInterpreter(object):
         allow_recursion -= 1
 
         should_return = False
+        # fails on (eg) if (...) stmt1; else stmt2;
         sub_statements = list(self._separate(stmt, ';')) or ['']
         expr = stmt = sub_statements.pop().strip()
         for sub_stmt in sub_statements:
@@ -371,25 +387,30 @@ class JSInterpreter(object):
         if expr[0] in _QUOTES:
             inner, outer = self._separate(expr, expr[0], 1)
             if expr[0] == '/':
-                flags, outer = self._regex_flags(outer)
-                inner = re.compile(inner[1:], flags=flags)  # , strict=True))
+                flags, outer = self.JS_RegExp.regex_flags(outer)
+                inner = self.JS_RegExp(inner[1:], flags=flags)
             else:
                 inner = json.loads(js_to_json(inner + expr[0]))  # , strict=True))
             if not outer:
                 return inner, should_return
             expr = self._named_object(local_vars, inner) + outer
 
-        if expr.startswith('new '):
-            obj = expr[4:]
-            if obj.startswith('Date('):
-                left, right = self._separate_at_paren(obj[4:])
-                expr = unified_timestamp(
-                    self.interpret_expression(left, local_vars, allow_recursion), False)
+        new_kw, _, obj = expr.partition('new ')
+        if not new_kw:
+            for klass, konstr in (('Date', lambda x: int(unified_timestamp(x, False) * 1000)),
+                                  ('RegExp', self.JS_RegExp),
+                                  ('Error', self.Exception)):
+                if not obj.startswith(klass + '('):
+                    continue
+                left, right = self._separate_at_paren(obj[len(klass):])
+                argvals = self.interpret_iter(left, local_vars, allow_recursion)
+                expr = konstr(*argvals)
                 if not expr:
-                    raise self.Exception('Failed to parse date {left!r}'.format(**locals()), expr=expr)
-                expr = self._dump(int(expr * 1000), local_vars) + right
+                    raise self.Exception('Failed to parse {klass} {left!r:.100}'.format(**locals()), expr=expr)
+                expr = self._dump(expr, local_vars) + right
+                break
             else:
-                raise self.Exception('Unsupported object {obj}'.format(**locals()), expr=expr)
+                raise self.Exception('Unsupported object {obj:.100}'.format(**locals()), expr=expr)
 
         if expr.startswith('void '):
             left = self.interpret_expression(expr[5:], local_vars, allow_recursion)
@@ -430,24 +451,45 @@ class JSInterpreter(object):
                 (?P<try>try)\s*\{|
                 (?P<if>if)\s*\(|
                 (?P<switch>switch)\s*\(|
-                (?P<for>for)\s*\(
+                (?P<for>for)\s*\(|
+                (?P<while>while)\s*\(
                 ''', expr)
         md = m.groupdict() if m else {}
         if md.get('if'):
             cndn, expr = self._separate_at_paren(expr[m.end() - 1:])
-            if_expr, expr = self._separate_at_paren(expr.lstrip())
-            # TODO: "else if" is not handled
+            if expr.startswith('{'):
+                if_expr, expr = self._separate_at_paren(expr)
+            else:
+                # may lose ... else ... because of ll.368-374
+                if_expr, expr = self._separate_at_paren(expr, delim=';')
             else_expr = None
-            m = re.match(r'else\s*{', expr)
+            m = re.match(r'else\s*(?P<block>\{)?', expr)
             if m:
-                else_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
+                if m.group('block'):
+                    else_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
+                else:
+                    # handle subset ... else if (...) {...} else ...
+                    # TODO: make interpret_statement do this properly, if possible
+                    exprs = list(self._separate(expr[m.end():], delim='}', max_split=2))
+                    if len(exprs) > 1:
+                        if re.match(r'\s*if\s*\(', exprs[0]) and re.match(r'\s*else\b', exprs[1]):
+                            else_expr = exprs[0] + '}' + exprs[1]
+                            expr = (exprs[2] + '}') if len(exprs) == 3 else None
+                        else:
+                            else_expr = exprs[0]
+                            exprs.append('')
+                            expr = '}'.join(exprs[1:])
+                    else:
+                        else_expr = exprs[0]
+                        expr = None
+                    else_expr = else_expr.lstrip() + '}'
             cndn = _js_ternary(self.interpret_expression(cndn, local_vars, allow_recursion))
             ret, should_abort = self.interpret_statement(
                 if_expr if cndn else else_expr, local_vars, allow_recursion)
             if should_abort:
                 return ret, True
 
-        if md.get('try'):
+        elif md.get('try'):
             try_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
             err = None
             try:
@@ -484,8 +526,8 @@ class JSInterpreter(object):
             if err:
                 raise err
 
-        elif md.get('for'):
-            constructor, remaining = self._separate_at_paren(expr[m.end() - 1:])
+        elif md.get('for') or md.get('while'):
+            init_or_cond, remaining = self._separate_at_paren(expr[m.end() - 1:])
             if remaining.startswith('{'):
                 body, expr = self._separate_at_paren(remaining)
             else:
@@ -496,11 +538,12 @@ class JSInterpreter(object):
                     body = 'switch(%s){%s}' % (switch_val, body)
                 else:
                     body, expr = remaining, ''
-            start, cndn, increment = self._separate(constructor, ';')
-            self.interpret_expression(start, local_vars, allow_recursion)
-            while True:
-                if not _js_ternary(self.interpret_expression(cndn, local_vars, allow_recursion)):
-                    break
+            if md.get('for'):
+                start, cndn, increment = self._separate(init_or_cond, ';')
+                self.interpret_expression(start, local_vars, allow_recursion)
+            else:
+                cndn, increment = init_or_cond, None
+            while _js_ternary(self.interpret_expression(cndn, local_vars, allow_recursion)):
                 try:
                     ret, should_abort = self.interpret_statement(body, local_vars, allow_recursion)
                     if should_abort:
@@ -509,7 +552,8 @@ class JSInterpreter(object):
                     break
                 except JS_Continue:
                     pass
-                self.interpret_expression(increment, local_vars, allow_recursion)
+                if increment:
+                    self.interpret_expression(increment, local_vars, allow_recursion)
 
         elif md.get('switch'):
             switch_val, remaining = self._separate_at_paren(expr[m.end() - 1:])
@@ -764,6 +808,10 @@ class JSInterpreter(object):
                     if idx >= len(obj):
                         return None
                     return ord(obj[idx])
+                elif member == 'replace':
+                    assertion(isinstance(obj, compat_str), 'must be applied on a string')
+                    assertion(len(argvals) == 2, 'takes exactly two arguments')
+                    return re.sub(argvals[0], argvals[1], obj)
 
                 idx = int(member) if isinstance(obj, list) else member
                 return obj[idx](argvals, allow_recursion=allow_recursion)
@@ -795,6 +843,10 @@ class JSInterpreter(object):
             raise self.Exception('Cannot return from an expression', expr)
         return ret
 
+    def interpret_iter(self, list_txt, local_vars, allow_recursion):
+        for v in self._separate(list_txt):
+            yield self.interpret_expression(v, local_vars, allow_recursion)
+
     def extract_object(self, objname):
         _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
         obj = {}

From 37cbdfa0e7c9d00d450af32dc9cdaf93cbfc4576 Mon Sep 17 00:00:00 2001
From: Brian Marks <bm1549@users.noreply.github.com>
Date: Thu, 2 Feb 2023 11:58:21 -0500
Subject: [PATCH 575/860] [americastestkitchen] Add support for downloading
 entire series (#31493)

Also
* support new sites and URL patterns
* back-port from yt-dlp

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/americastestkitchen.py | 115 +++++++++++++++-----
 1 file changed, 88 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/americastestkitchen.py b/youtube_dl/extractor/americastestkitchen.py
index be960c0f9..08d3604e9 100644
--- a/youtube_dl/extractor/americastestkitchen.py
+++ b/youtube_dl/extractor/americastestkitchen.py
@@ -15,7 +15,7 @@ from ..utils import (
 
 
 class AmericasTestKitchenIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?:americastestkitchen|cooks(?:country|illustrated))\.com/(?P<resource_type>episode|videos)/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?(?:americastestkitchen|cooks(?:country|illustrated))\.com/(?:cooks(?:country|illustrated)/)?(?P<resource_type>episode|videos)/(?P<id>\d+)'
     _TESTS = [{
         'url': 'https://www.americastestkitchen.com/episode/582-weeknight-japanese-suppers',
         'md5': 'b861c3e365ac38ad319cfd509c30577f',
@@ -23,15 +23,20 @@ class AmericasTestKitchenIE(InfoExtractor):
             'id': '5b400b9ee338f922cb06450c',
             'title': 'Japanese Suppers',
             'ext': 'mp4',
+            'display_id': 'weeknight-japanese-suppers',
             'description': 'md5:64e606bfee910627efc4b5f050de92b3',
-            'thumbnail': r're:^https?://',
-            'timestamp': 1523318400,
-            'upload_date': '20180410',
-            'release_date': '20180410',
+            'timestamp': 1523304000,
+            'upload_date': '20180409',
+            'release_date': '20180409',
             'series': "America's Test Kitchen",
+            'season': 'Season 18',
             'season_number': 18,
             'episode': 'Japanese Suppers',
             'episode_number': 15,
+            'duration': 1376,
+            'thumbnail': r're:^https?://',
+            'average_rating': 0,
+            'view_count': int,
         },
         'params': {
             'skip_download': True,
@@ -44,15 +49,20 @@ class AmericasTestKitchenIE(InfoExtractor):
             'id': '5fbe8c61bda2010001c6763b',
             'title': 'Simple Chicken Dinner',
             'ext': 'mp4',
+            'display_id': 'atktv_2103_simple-chicken-dinner_full-episode_web-mp4',
             'description': 'md5:eb68737cc2fd4c26ca7db30139d109e7',
-            'thumbnail': r're:^https?://',
-            'timestamp': 1610755200,
-            'upload_date': '20210116',
-            'release_date': '20210116',
+            'timestamp': 1610737200,
+            'upload_date': '20210115',
+            'release_date': '20210115',
             'series': "America's Test Kitchen",
+            'season': 'Season 21',
             'season_number': 21,
             'episode': 'Simple Chicken Dinner',
             'episode_number': 3,
+            'duration': 1397,
+            'thumbnail': r're:^https?://',
+            'view_count': int,
+            'average_rating': 0,
         },
         'params': {
             'skip_download': True,
@@ -60,6 +70,12 @@ class AmericasTestKitchenIE(InfoExtractor):
     }, {
         'url': 'https://www.americastestkitchen.com/videos/3420-pan-seared-salmon',
         'only_matching': True,
+    }, {
+        'url': 'https://www.americastestkitchen.com/cookscountry/episode/564-when-only-chocolate-will-do',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.americastestkitchen.com/cooksillustrated/videos/4478-beef-wellington',
+        'only_matching': True,
     }, {
         'url': 'https://www.cookscountry.com/episode/564-when-only-chocolate-will-do',
         'only_matching': True,
@@ -94,7 +110,7 @@ class AmericasTestKitchenIE(InfoExtractor):
 
 
 class AmericasTestKitchenSeasonIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?(?P<show>americastestkitchen|cookscountry)\.com/episodes/browse/season_(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?(?P<show>americastestkitchen|(?P<cooks>cooks(?:country|illustrated)))\.com(?:(?:/(?P<show2>cooks(?:country|illustrated)))?(?:/?$|(?<!ated)(?<!ated\.com)/episodes/browse/season_(?P<season>\d+)))'
     _TESTS = [{
         # ATK Season
         'url': 'https://www.americastestkitchen.com/episodes/browse/season_1',
@@ -105,48 +121,93 @@ class AmericasTestKitchenSeasonIE(InfoExtractor):
         'playlist_count': 13,
     }, {
         # Cooks Country Season
-        'url': 'https://www.cookscountry.com/episodes/browse/season_12',
+        'url': 'https://www.americastestkitchen.com/cookscountry/episodes/browse/season_12',
         'info_dict': {
             'id': 'season_12',
             'title': 'Season 12',
         },
         'playlist_count': 13,
+    }, {
+        # America's Test Kitchen Series
+        'url': 'https://www.americastestkitchen.com/',
+        'info_dict': {
+            'id': 'americastestkitchen',
+            'title': 'America\'s Test Kitchen',
+        },
+        'playlist_count': 558,
+    }, {
+        # Cooks Country Series
+        'url': 'https://www.americastestkitchen.com/cookscountry',
+        'info_dict': {
+            'id': 'cookscountry',
+            'title': 'Cook\'s Country',
+        },
+        'playlist_count': 199,
+    }, {
+        'url': 'https://www.americastestkitchen.com/cookscountry/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.cookscountry.com/episodes/browse/season_12',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.cookscountry.com',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.americastestkitchen.com/cooksillustrated/',
+        'only_matching': True,
+    }, {
+        'url': 'https://www.cooksillustrated.com',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
-        show_name, season_number = re.match(self._VALID_URL, url).groups()
-        season_number = int(season_number)
+        match = re.match(self._VALID_URL, url).groupdict()
+        show = match.get('show2')
+        show_path = ('/' + show) if show else ''
+        show = show or match['show']
+        season_number = int_or_none(match.get('season'))
 
-        slug = 'atk' if show_name == 'americastestkitchen' else 'cco'
+        slug, title = {
+            'americastestkitchen': ('atk', 'America\'s Test Kitchen'),
+            'cookscountry': ('cco', 'Cook\'s Country'),
+            'cooksillustrated': ('cio', 'Cook\'s Illustrated'),
+        }[show]
 
-        season = 'Season %d' % season_number
+        facet_filters = [
+            'search_document_klass:episode',
+            'search_show_slug:' + slug,
+        ]
+
+        if season_number:
+            playlist_id = 'season_%d' % season_number
+            playlist_title = 'Season %d' % season_number
+            facet_filters.append('search_season_list:' + playlist_title)
+        else:
+            playlist_id = show
+            playlist_title = title
 
         season_search = self._download_json(
             'https://y1fnzxui30-dsn.algolia.net/1/indexes/everest_search_%s_season_desc_production' % slug,
-            season, headers={
-                'Origin': 'https://www.%s.com' % show_name,
+            playlist_id, headers={
+                'Origin': 'https://www.americastestkitchen.com',
                 'X-Algolia-API-Key': '8d504d0099ed27c1b73708d22871d805',
                 'X-Algolia-Application-Id': 'Y1FNZXUI30',
             }, query={
-                'facetFilters': json.dumps([
-                    'search_season_list:' + season,
-                    'search_document_klass:episode',
-                    'search_show_slug:' + slug,
-                ]),
-                'attributesToRetrieve': 'description,search_%s_episode_number,search_document_date,search_url,title' % slug,
+                'facetFilters': json.dumps(facet_filters),
+                'attributesToRetrieve': 'description,search_%s_episode_number,search_document_date,search_url,title,search_atk_episode_season' % slug,
                 'attributesToHighlight': '',
                 'hitsPerPage': 1000,
             })
 
         def entries():
             for episode in (season_search.get('hits') or []):
-                search_url = episode.get('search_url')
+                search_url = episode.get('search_url')  # always formatted like '/episode/123-title-of-episode'
                 if not search_url:
                     continue
                 yield {
                     '_type': 'url',
-                    'url': 'https://www.%s.com%s' % (show_name, search_url),
-                    'id': try_get(episode, lambda e: e['objectID'].split('_')[-1]),
+                    'url': 'https://www.americastestkitchen.com%s%s' % (show_path, search_url),
+                    'id': try_get(episode, lambda e: e['objectID'].rsplit('_', 1)[-1]),
                     'title': episode.get('title'),
                     'description': episode.get('description'),
                     'timestamp': unified_timestamp(episode.get('search_document_date')),
@@ -156,4 +217,4 @@ class AmericasTestKitchenSeasonIE(InfoExtractor):
                 }
 
         return self.playlist_result(
-            entries(), 'season_%d' % season_number, season)
+            entries(), playlist_id, playlist_title)

From 297fbff23b347612a5f6002b40adba9dfad85413 Mon Sep 17 00:00:00 2001
From: Rodrigo Dias <roycocup@users.noreply.github.com>
Date: Thu, 2 Feb 2023 17:10:09 +0000
Subject: [PATCH 576/860] [doc] Fixed typo appearing to promise an example
 (#31489)

Resolves #31425

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index cd888c731..6e07ddb1c 100644
--- a/README.md
+++ b/README.md
@@ -632,7 +632,7 @@ To use percent literals in an output template use `%%`. To output to stdout use
 
 The current default template is `%(title)s-%(id)s.%(ext)s`.
 
-In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title:
+In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title.
 
 #### Output template and Windows batch files
 

From 807e593a32a1ace8fa0be8129fc5071d86516c99 Mon Sep 17 00:00:00 2001
From: JChris246 <43832407+JChris246@users.noreply.github.com>
Date: Thu, 2 Feb 2023 13:12:36 -0400
Subject: [PATCH 577/860] [cammodels] fix and improve extractor (#31453)

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/cammodels.py | 34 +++++++++----------------------
 1 file changed, 10 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/cammodels.py b/youtube_dl/extractor/cammodels.py
index 1eb81b75e..d2e860b24 100644
--- a/youtube_dl/extractor/cammodels.py
+++ b/youtube_dl/extractor/cammodels.py
@@ -3,7 +3,6 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 from ..utils import (
-    ExtractorError,
     int_or_none,
     url_or_none,
 )
@@ -20,32 +19,11 @@ class CamModelsIE(InfoExtractor):
     def _real_extract(self, url):
         user_id = self._match_id(url)
 
-        webpage = self._download_webpage(
-            url, user_id, headers=self.geo_verification_headers())
-
-        manifest_root = self._html_search_regex(
-            r'manifestUrlRoot=([^&\']+)', webpage, 'manifest', default=None)
-
-        if not manifest_root:
-            ERRORS = (
-                ("I'm offline, but let's stay connected", 'This user is currently offline'),
-                ('in a private show', 'This user is in a private show'),
-                ('is currently performing LIVE', 'This model is currently performing live'),
-            )
-            for pattern, message in ERRORS:
-                if pattern in webpage:
-                    error = message
-                    expected = True
-                    break
-            else:
-                error = 'Unable to find manifest URL root'
-                expected = False
-            raise ExtractorError(error, expected=expected)
-
         manifest = self._download_json(
-            '%s%s.json' % (manifest_root, user_id), user_id)
+            'https://manifest-server.naiadsystems.com/live/s:%s.json' % user_id, user_id)
 
         formats = []
+        thumbnails = []
         for format_id, format_dict in manifest['formats'].items():
             if not isinstance(format_dict, dict):
                 continue
@@ -85,6 +63,13 @@ class CamModelsIE(InfoExtractor):
                         'preference': -1,
                     })
                 else:
+                    if format_id == 'jpeg':
+                        thumbnails.append({
+                            'url': f['url'],
+                            'width': f['width'],
+                            'height': f['height'],
+                            'format_id': f['format_id'],
+                        })
                     continue
                 formats.append(f)
         self._sort_formats(formats)
@@ -92,6 +77,7 @@ class CamModelsIE(InfoExtractor):
         return {
             'id': user_id,
             'title': self._live_title(user_id),
+            'thumbnails': thumbnails,
             'is_live': True,
             'formats': formats,
             'age_limit': 18

From e9611a2a3603ee201d0c1ba99e8bfd8ec1e697cd Mon Sep 17 00:00:00 2001
From: Leon Etienne <40911701+Leonetienne@users.noreply.github.com>
Date: Thu, 2 Feb 2023 18:13:39 +0100
Subject: [PATCH 578/860] [pr0gramm] implement InfoExtractor, Resolves #31433
 (#31434)

* [pr0gramm] implement infoextractor

* [pr0gramm] remove misplaced comment, uncapture regex-group

* [pr0gramm]: specify utf-8 coding

* [pr0gramm]: add trailing comma to lists for maintainability

* [pr0gramm]: ie only sets upload_date attribute

* [pr0gramm]: add video_id to title

* [pr0gramm]: more forgiving _valid_url regex

* [pr0gramm]: add uploader to title, if set

* Discriminate URL pattern

---------

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/extractors.py |   4 ++
 youtube_dl/extractor/pr0gramm.py   | 105 +++++++++++++++++++++++++++++
 2 files changed, 109 insertions(+)
 create mode 100644 youtube_dl/extractor/pr0gramm.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 947cbe8fd..cf0388ed2 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1667,3 +1667,7 @@ from .zingmp3 import (
 )
 from .zoom import ZoomIE
 from .zype import ZypeIE
+from .pr0gramm import (
+    Pr0grammIE,
+    Pr0grammStaticIE,
+)
diff --git a/youtube_dl/extractor/pr0gramm.py b/youtube_dl/extractor/pr0gramm.py
new file mode 100644
index 000000000..b68224fd5
--- /dev/null
+++ b/youtube_dl/extractor/pr0gramm.py
@@ -0,0 +1,105 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+import re
+from ..utils import (
+    merge_dicts,
+)
+
+
+class Pr0grammStaticIE(InfoExtractor):
+    # Possible urls:
+    # https://pr0gramm.com/static/5466437
+    _VALID_URL = r'https?://pr0gramm\.com/static/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://pr0gramm.com/static/5466437',
+        'md5': '52fa540d70d3edc286846f8ca85938aa',
+        'info_dict': {
+            'id': '5466437',
+            'ext': 'mp4',
+            'title': 'pr0gramm-5466437 by g11st',
+            'uploader': 'g11st',
+            'upload_date': '20221221',
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        # Fetch media sources
+        entries = self._parse_html5_media_entries(url, webpage, video_id)
+        media_info = entries[0]
+
+        # this raises if there are no formats
+        self._sort_formats(media_info.get('formats') or [])
+
+        # Fetch author
+        uploader = self._html_search_regex(r'by\W+([\w-]+)\W+', webpage, 'uploader')
+
+        # Fetch approx upload timestamp from filename
+        # Have None-defaults in case the extraction fails
+        uploadDay = None
+        uploadMon = None
+        uploadYear = None
+        uploadTimestr = None
+        # (//img.pr0gramm.com/2022/12/21/62ae8aa5e2da0ebf.mp4)
+        m = re.search(r'//img\.pr0gramm\.com/(?P<year>[\d]+)/(?P<mon>[\d]+)/(?P<day>[\d]+)/\w+\.\w{,4}', webpage)
+
+        if (m):
+            # Up to a day of accuracy should suffice...
+            uploadDay = m.groupdict().get('day')
+            uploadMon = m.groupdict().get('mon')
+            uploadYear = m.groupdict().get('year')
+            uploadTimestr = uploadYear + uploadMon + uploadDay
+
+        return merge_dicts({
+            'id': video_id,
+            'title': 'pr0gramm-%s%s' % (video_id, (' by ' + uploader) if uploader else ''),
+            'uploader': uploader,
+            'upload_date': uploadTimestr
+        }, media_info)
+
+
+# This extractor is for the primary url (used for sharing, and appears in the
+# location bar) Since this page loads the DOM via JS, yt-dl can't find any
+# video information here. So let's redirect to a compatibility version of
+# the site, which does contain the <video>-element  by itself,  without requiring
+# js to be ran.
+class Pr0grammIE(InfoExtractor):
+    # Possible urls:
+    # https://pr0gramm.com/new/546637
+    # https://pr0gramm.com/new/video/546637
+    # https://pr0gramm.com/top/546637
+    # https://pr0gramm.com/top/video/546637
+    # https://pr0gramm.com/user/g11st/uploads/5466437
+    # https://pr0gramm.com/user/froschler/dafur-ist-man-hier/5091290
+    # https://pr0gramm.com/user/froschler/reinziehen-1elf/5232030
+    # https://pr0gramm.com/user/froschler/1elf/5232030
+    # https://pr0gramm.com/new/5495710:comment62621020 <- this is not the id!
+    # https://pr0gramm.com/top/fruher war alles damals/5498175
+
+    _VALID_URL = r'https?:\/\/pr0gramm\.com\/(?!static/\d+).+?\/(?P<id>[\d]+)(:|$)'
+    _TEST = {
+        'url': 'https://pr0gramm.com/new/video/5466437',
+        'info_dict': {
+            'id': '5466437',
+            'ext': 'mp4',
+            'title': 'pr0gramm-5466437 by g11st',
+            'uploader': 'g11st',
+            'upload_date': '20221221',
+        }
+    }
+
+    def _generic_title():
+        return "oof"
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        return self.url_result(
+            'https://pr0gramm.com/static/' + video_id,
+            video_id=video_id,
+            ie=Pr0grammStaticIE.ie_key())

From 98b0cf1cd05c493eae0f37aaa599d25d2848c0b0 Mon Sep 17 00:00:00 2001
From: Ruowang Sun <91006887+JohnnySunUmich@users.noreply.github.com>
Date: Thu, 2 Feb 2023 12:21:05 -0500
Subject: [PATCH 579/860] [Callin] Add new extractor (#31414)

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/callin.py     | 74 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 75 insertions(+)
 create mode 100644 youtube_dl/extractor/callin.py

diff --git a/youtube_dl/extractor/callin.py b/youtube_dl/extractor/callin.py
new file mode 100644
index 000000000..341be479f
--- /dev/null
+++ b/youtube_dl/extractor/callin.py
@@ -0,0 +1,74 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    ExtractorError,
+    traverse_obj,
+    try_get,
+)
+
+
+class CallinIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?callin\.com/episode/(?:[^/#?-]+-)*(?P<id>[^/#?-]+)'
+    _TESTS = [{
+        'url': 'https://www.callin.com/episode/fcc-commissioner-brendan-carr-on-elons-PrumRdSQJW',
+        'md5': '14ede27ee2c957b7e4db93140fc0745c',
+        'info_dict': {
+            'id': 'PrumRdSQJW',
+            'ext': 'mp4',
+            'title': 'FCC Commissioner Brendan Carr on Elon’s Starlink',
+            'description': 'Or, why the government doesn’t like SpaceX',
+            'channel': 'The Pull Request',
+            'channel_url': 'https://callin.com/show/the-pull-request-ucnDJmEKAa',
+        }
+    }, {
+        'url': 'https://www.callin.com/episode/episode-81-elites-melt-down-over-student-debt-lzxMidUnjA',
+        'md5': '16f704ddbf82a27e3930533b12062f07',
+        'info_dict': {
+            'id': 'lzxMidUnjA',
+            'ext': 'mp4',
+            'title': 'Episode 81- Elites MELT DOWN over Student Debt Victory? Rumble in NYC?',
+            'description': 'Let’s talk todays episode about the primary election shake up in NYC and the elites melting down over student debt cancelation.',
+            'channel': 'The DEBRIEF With Briahna Joy Gray',
+            'channel_url': 'https://callin.com/show/the-debrief-with-briahna-joy-gray-siiFDzGegm',
+        }
+    }]
+
+    def _search_nextjs_data(self, webpage, video_id, transform_source=None, fatal=True, **kw):
+        return self._parse_json(
+            self._search_regex(
+                r'(?s)<script[^>]+id=[\'"]__NEXT_DATA__[\'"][^>]*>([^<]+)</script>',
+                webpage, 'next.js data', fatal=fatal, **kw),
+            video_id, transform_source=transform_source, fatal=fatal)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        next_data = self._search_nextjs_data(webpage, video_id)
+        episode = traverse_obj(next_data, ('props', 'pageProps', 'episode'), expected_type=dict)
+        if not episode:
+            raise ExtractorError('Failed to find episode data')
+
+        title = episode.get('title') or self._og_search_title(webpage)
+        description = episode.get('description') or self._og_search_description(webpage)
+
+        formats = []
+        formats.extend(self._extract_m3u8_formats(
+            episode.get('m3u8'), video_id, 'mp4',
+            entry_protocol='m3u8_native', fatal=False))
+        self._sort_formats(formats)
+
+        channel = try_get(episode, lambda x: x['show']['title'], compat_str)
+        channel_url = try_get(episode, lambda x: x['show']['linkObj']['resourceUrl'], compat_str)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'formats': formats,
+            'channel': channel,
+            'channel_url': channel_url,
+        }
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index cf0388ed2..f7bb4042f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -158,6 +158,7 @@ from .businessinsider import BusinessInsiderIE
 from .buzzfeed import BuzzFeedIE
 from .byutv import BYUtvIE
 from .c56 import C56IE
+from .callin import CallinIE
 from .camdemy import (
     CamdemyIE,
     CamdemyFolderIE

From 6d829d811932b24be4d3cc8b6c1e0d46c2b1566c Mon Sep 17 00:00:00 2001
From: zhangeric-15 <71106422+zhangeric-15@users.noreply.github.com>
Date: Thu, 2 Feb 2023 12:26:31 -0500
Subject: [PATCH 580/860] [YouTube] Fix not finding videos listed under a
 channel's "shorts" subpage.  (#31409)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Resolves #31336

Co-authored-by: Jouni Järvinen <rautamiekka@users.noreply.github.com>
Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/youtube.py | 27 +++++++++++++++++++++++++--
 1 file changed, 25 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3d12e2e4a..28fdb086a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -315,7 +315,8 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         title = try_get(
             renderer,
             (lambda x: x['title']['runs'][0]['text'],
-             lambda x: x['title']['simpleText']), compat_str)
+             lambda x: x['title']['simpleText'],
+             lambda x: x['headline']['simpleText']), compat_str)
         description = try_get(
             renderer, lambda x: x['descriptionSnippet']['runs'][0]['text'],
             compat_str)
@@ -2207,6 +2208,24 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
     IE_NAME = 'youtube:tab'
 
     _TESTS = [{
+        # Shorts
+        'url': 'https://www.youtube.com/@SuperCooperShorts/shorts',
+        'playlist_mincount': 5,
+        'info_dict': {
+            'description': 'Short clips from Super Cooper Sundays!',
+            'id': 'UCKMA8kHZ8bPYpnMNaUSxfEQ',
+            'title': 'Super Cooper Shorts - Shorts',
+        }
+    }, {
+        # Channel that does not have a Shorts tab. Test should just download videos on Home tab instead
+        'url': 'https://www.youtube.com/@emergencyawesome/shorts',
+        'info_dict': {
+            'description': 'md5:592c080c06fef4de3c902c4a8eecd850',
+            'id': 'UCDiFRMQWpcp8_KD4vwIVicw',
+            'title': 'Emergency Awesome - Home',
+        },
+        'playlist_mincount': 5,
+    }, {
         # playlists, multipage
         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
         'playlist_mincount': 94,
@@ -2680,7 +2699,11 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
 
     def _rich_grid_entries(self, contents):
         for content in contents:
-            video_renderer = try_get(content, lambda x: x['richItemRenderer']['content']['videoRenderer'], dict)
+            video_renderer = try_get(
+                content,
+                (lambda x: x['richItemRenderer']['content']['videoRenderer'],
+                 lambda x: x['richItemRenderer']['content']['reelItemRenderer']),
+                dict)
             if video_renderer:
                 entry = self._video_entry(video_renderer)
                 if entry:

From be3392a0d491af81f353b4372d47d589fda54b0c Mon Sep 17 00:00:00 2001
From: Epsilonator <28658223+clueless-skywatcher@users.noreply.github.com>
Date: Thu, 2 Feb 2023 23:03:09 +0530
Subject: [PATCH 581/860] [Blerp] Add new extractor (#31398)

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/blerp.py      | 173 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |   1 +
 2 files changed, 174 insertions(+)
 create mode 100644 youtube_dl/extractor/blerp.py

diff --git a/youtube_dl/extractor/blerp.py b/youtube_dl/extractor/blerp.py
new file mode 100644
index 000000000..355daef6e
--- /dev/null
+++ b/youtube_dl/extractor/blerp.py
@@ -0,0 +1,173 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import json
+
+from ..utils import (
+    strip_or_none,
+    traverse_obj,
+)
+from .common import InfoExtractor
+
+
+class BlerpIE(InfoExtractor):
+    IE_NAME = 'blerp'
+    _VALID_URL = r'https?://(?:www\.)?blerp\.com/soundbites/(?P<id>[0-9a-zA-Z]+)'
+    _TESTS = [{
+        'url': 'https://blerp.com/soundbites/6320fe8745636cb4dd677a5a',
+        'info_dict': {
+            'id': '6320fe8745636cb4dd677a5a',
+            'title': 'Samsung Galaxy S8 Over the Horizon Ringtone 2016',
+            'uploader': 'luminousaj',
+            'uploader_id': '5fb81e51aa66ae000c395478',
+            'ext': 'mp3',
+            'tags': ['samsung', 'galaxy', 's8', 'over the horizon', '2016', 'ringtone'],
+        }
+    }, {
+        'url': 'https://blerp.com/soundbites/5bc94ef4796001000498429f',
+        'info_dict': {
+            'id': '5bc94ef4796001000498429f',
+            'title': 'Yee',
+            'uploader': '179617322678353920',
+            'uploader_id': '5ba99cf71386730004552c42',
+            'ext': 'mp3',
+            'tags': ['YEE', 'YEET', 'wo ha haah catchy tune yee', 'yee']
+        }
+    }]
+
+    _GRAPHQL_OPERATIONNAME = "webBitePageGetBite"
+    _GRAPHQL_QUERY = (
+        '''query webBitePageGetBite($_id: MongoID!) {
+            web {
+                biteById(_id: $_id) {
+                    ...bitePageFrag
+                    __typename
+                }
+                __typename
+            }
+        }
+
+        fragment bitePageFrag on Bite {
+            _id
+            title
+            userKeywords
+            keywords
+            color
+            visibility
+            isPremium
+            owned
+            price
+            extraReview
+            isAudioExists
+            image {
+                filename
+                original {
+                    url
+                    __typename
+                }
+                __typename
+            }
+            userReactions {
+                _id
+                reactions
+                createdAt
+                __typename
+            }
+            topReactions
+            totalSaveCount
+            saved
+            blerpLibraryType
+            license
+            licenseMetaData
+            playCount
+            totalShareCount
+            totalFavoriteCount
+            totalAddedToBoardCount
+            userCategory
+            userAudioQuality
+            audioCreationState
+            transcription
+            userTranscription
+            description
+            createdAt
+            updatedAt
+            author
+            listingType
+            ownerObject {
+                _id
+                username
+                profileImage {
+                    filename
+                    original {
+                        url
+                        __typename
+                    }
+                    __typename
+                }
+                __typename
+            }
+            transcription
+            favorited
+            visibility
+            isCurated
+            sourceUrl
+            audienceRating
+            strictAudienceRating
+            ownerId
+            reportObject {
+                reportedContentStatus
+                __typename
+            }
+            giphy {
+                mp4
+                gif
+                __typename
+            }
+            audio {
+                filename
+                original {
+                    url
+                    __typename
+                }
+                mp3 {
+                    url
+                    __typename
+                }
+                __typename
+            }
+            __typename
+        }
+
+        ''')
+
+    def _real_extract(self, url):
+        audio_id = self._match_id(url)
+
+        data = {
+            'operationName': self._GRAPHQL_OPERATIONNAME,
+            'query': self._GRAPHQL_QUERY,
+            'variables': {
+                '_id': audio_id
+            }
+        }
+
+        headers = {
+            'Content-Type': 'application/json'
+        }
+
+        json_result = self._download_json('https://api.blerp.com/graphql',
+                                          audio_id, data=json.dumps(data).encode('utf-8'), headers=headers)
+
+        bite_json = json_result['data']['web']['biteById']
+
+        info_dict = {
+            'id': bite_json['_id'],
+            'url': bite_json['audio']['mp3']['url'],
+            'title': bite_json['title'],
+            'uploader': traverse_obj(bite_json, ('ownerObject', 'username'), expected_type=strip_or_none),
+            'uploader_id': traverse_obj(bite_json, ('ownerObject', '_id'), expected_type=strip_or_none),
+            'ext': 'mp3',
+            'tags': list(filter(None, map(strip_or_none, (traverse_obj(bite_json, 'userKeywords', expected_type=list) or []))) or None)
+        }
+
+        return info_dict
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index f7bb4042f..b8db4c818 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -138,6 +138,7 @@ from .bleacherreport import (
     BleacherReportIE,
     BleacherReportCMSIE,
 )
+from .blerp import BlerpIE
 from .bloomberg import BloombergIE
 from .bokecc import BokeCCIE
 from .bongacams import BongaCamsIE

From bc6f94e459694f541a2a1078fad59b02f2fc9d4c Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 2 Feb 2023 23:19:03 +0000
Subject: [PATCH 582/860] [FIFA] Back-port extractor from yt-dlp (#31385)

---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/fifa.py       | 101 +++++++++++++++++++++++++++++
 2 files changed, 102 insertions(+)
 create mode 100644 youtube_dl/extractor/fifa.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b8db4c818..31a3e588e 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -376,6 +376,7 @@ from .fc2 import (
     FC2EmbedIE,
 )
 from .fczenit import FczenitIE
+from .fifa import FifaIE
 from .filmon import (
     FilmOnIE,
     FilmOnChannelIE,
diff --git a/youtube_dl/extractor/fifa.py b/youtube_dl/extractor/fifa.py
new file mode 100644
index 000000000..15157774e
--- /dev/null
+++ b/youtube_dl/extractor/fifa.py
@@ -0,0 +1,101 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+
+from ..utils import (
+    int_or_none,
+    traverse_obj,
+    unified_timestamp,
+)
+
+if not callable(getattr(InfoExtractor, '_match_valid_url', None)):
+
+    BaseInfoExtractor = InfoExtractor
+
+    import re
+
+    class InfoExtractor(BaseInfoExtractor):
+
+        @classmethod
+        def _match_valid_url(cls, url):
+            return re.match(cls._VALID_URL, url)
+
+
+class FifaIE(InfoExtractor):
+    _VALID_URL = r'https?://www.fifa.com/fifaplus/(?P<locale>\w{2})/watch/([^#?]+/)?(?P<id>\w+)'
+    _TESTS = [{
+        'url': 'https://www.fifa.com/fifaplus/en/watch/7on10qPcnyLajDDU3ntg6y',
+        'info_dict': {
+            'id': '7on10qPcnyLajDDU3ntg6y',
+            'title': 'Italy v France | Final | 2006 FIFA World Cup Germany™ | Full Match Replay',
+            'description': 'md5:f4520d0ee80529c8ba4134a7d692ff8b',
+            'ext': 'mp4',
+            'categories': ['FIFA Tournaments'],
+            'thumbnail': 'https://digitalhub.fifa.com/transform/135e2656-3a51-407b-8810-6c34bec5b59b/FMR_2006_Italy_France_Final_Hero',
+            'duration': 8165,
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.fifa.com/fifaplus/pt/watch/1cg5r5Qt6Qt12ilkDgb1sV',
+        'info_dict': {
+            'id': '1cg5r5Qt6Qt12ilkDgb1sV',
+            'title': 'Brazil v Germany | Semi-finals | 2014 FIFA World Cup Brazil™ | Extended Highlights',
+            'description': 'md5:d908c74ee66322b804ae2e521b02a855',
+            'ext': 'mp4',
+            'categories': ['FIFA Tournaments', 'Highlights'],
+            'thumbnail': 'https://digitalhub.fifa.com/transform/d8fe6f61-276d-4a73-a7fe-6878a35fd082/FIFAPLS_100EXTHL_2014BRAvGER_TMB',
+            'duration': 902,
+            'release_timestamp': 1404777600,
+            'release_date': '20140708',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }, {
+        'url': 'https://www.fifa.com/fifaplus/fr/watch/3C6gQH9C2DLwzNx7BMRQdp',
+        'info_dict': {
+            'id': '3C6gQH9C2DLwzNx7BMRQdp',
+            'title': 'Josimar goal against Northern Ireland | Classic Goals',
+            'description': 'md5:cbe7e7bb52f603c9f1fe9a4780fe983b',
+            'ext': 'mp4',
+            'categories': ['FIFA Tournaments', 'Goal'],
+            'duration': 28,
+            'thumbnail': 'https://digitalhub.fifa.com/transform/f9301391-f8d9-48b5-823e-c093ac5e3e11/CG_MEN_1986_JOSIMAR',
+        },
+        'params': {'skip_download': 'm3u8'},
+    }]
+
+    def _real_extract(self, url):
+        video_id, locale = self._match_valid_url(url).group('id', 'locale')
+        webpage = self._download_webpage(url, video_id)
+
+        preconnect_link = self._search_regex(
+            r'<link\b[^>]+\brel\s*=\s*"preconnect"[^>]+href\s*=\s*"([^"]+)"', webpage, 'Preconnect Link')
+
+        video_details = self._download_json(
+            '{preconnect_link}/sections/videoDetails/{video_id}'.format(**locals()), video_id, 'Downloading Video Details', fatal=False)
+
+        preplay_parameters = self._download_json(
+            '{preconnect_link}/videoPlayerData/{video_id}'.format(**locals()), video_id, 'Downloading Preplay Parameters')['preplayParameters']
+
+        content_data = self._download_json(
+            # 1. query string is expected to be sent as-is
+            # 2. `sig` must be appended
+            # 3. if absent, the call appears to work but the manifest is bad (404)
+            'https://content.uplynk.com/preplay/{contentId}/multiple.json?{queryStr}&sig={signature}'.format(**preplay_parameters),
+            video_id, 'Downloading Content Data')
+
+        # formats, subtitles = self._extract_m3u8_formats_and_subtitles(content_data['playURL'], video_id)
+        formats, subtitles = self._extract_m3u8_formats(content_data['playURL'], video_id, ext='mp4', entry_protocol='m3u8_native'), None
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': video_details['title'],
+            'description': video_details.get('description'),
+            'duration': int_or_none(video_details.get('duration')),
+            'release_timestamp': unified_timestamp(video_details.get('dateOfRelease')),
+            'categories': traverse_obj(video_details, (('videoCategory', 'videoSubcategory'),)),
+            'thumbnail': traverse_obj(video_details, ('backgroundImage', 'src')),
+            'formats': formats,
+            'subtitles': subtitles,
+        }

From f316f5d4e391ca40273bce65c67bedc16ae99172 Mon Sep 17 00:00:00 2001
From: afterdelight <39585663+afterdelight@users.noreply.github.com>
Date: Fri, 3 Feb 2023 06:20:14 +0700
Subject: [PATCH 583/860] [xhamster] add support for new domain xhvid.com
 (#31370)

---
 youtube_dl/extractor/xhamster.py | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/xhamster.py b/youtube_dl/extractor/xhamster.py
index f764021ba..e17947fc6 100644
--- a/youtube_dl/extractor/xhamster.py
+++ b/youtube_dl/extractor/xhamster.py
@@ -24,7 +24,7 @@ from ..utils import (
 
 
 class XHamsterIE(InfoExtractor):
-    _DOMAINS = r'(?:xhamster\.(?:com|one|desi)|xhms\.pro|xhamster\d+\.com|xhday\.com)'
+    _DOMAINS = r'(?:xhamster\.(?:com|one|desi)|xhms\.pro|xhamster\d+\.com|xhday\.com|xhvid\.com)'
     _VALID_URL = r'''(?x)
                     https?://
                         (?:.+?\.)?%s/
@@ -123,6 +123,9 @@ class XHamsterIE(InfoExtractor):
     }, {
         'url': 'https://xhday.com/videos/strapless-threesome-xhh7yVf',
         'only_matching': True,
+    }, {
+        'url': 'https://xhvid.com/videos/lk-mm-xhc6wn6',
+        'only_matching': True,
     }]
 
     def _real_extract(self, url):
@@ -433,6 +436,9 @@ class XHamsterUserIE(InfoExtractor):
     }, {
         'url': 'https://xhday.com/users/mobhunter',
         'only_matching': True,
+    }, {
+        'url': 'https://xhvid.com/users/pelushe21',
+        'only_matching': True,
     }]
 
     def _entries(self, user_id):

From 9d17948b5a1cc48bd526b1163292415577131c31 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 2 Feb 2023 23:25:44 +0000
Subject: [PATCH 584/860] [myvideoge] Add new extractor (#31360)

NB download tests on CI servers blocked

Co-authored-by: Alfonso Solbes <fonk666@gmail.com>
---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/myvideoge.py  | 87 ++++++++++++++++++++++++++++++
 2 files changed, 88 insertions(+)
 create mode 100644 youtube_dl/extractor/myvideoge.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 31a3e588e..96b27b179 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -728,6 +728,7 @@ from .myvi import (
     MyviIE,
     MyviEmbedIE,
 )
+from .myvideoge import MyVideoGeIE
 from .myvidster import MyVidsterIE
 from .nationalgeographic import (
     NationalGeographicVideoIE,
diff --git a/youtube_dl/extractor/myvideoge.py b/youtube_dl/extractor/myvideoge.py
new file mode 100644
index 000000000..efbfda7a6
--- /dev/null
+++ b/youtube_dl/extractor/myvideoge.py
@@ -0,0 +1,87 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    get_element_by_id,
+    get_element_by_class,
+    int_or_none,
+    js_to_json,
+    MONTH_NAMES,
+    qualities,
+    unified_strdate,
+)
+
+
+class MyVideoGeIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?myvideo\.ge/v/(?P<id>[0-9]+)'
+    _TEST = {
+        'url': 'https://www.myvideo.ge/v/3941048',
+        'md5': '8c192a7d2b15454ba4f29dc9c9a52ea9',
+        'info_dict': {
+            'id': '3941048',
+            'ext': 'mp4',
+            'title': 'The best prikol',
+            'upload_date': '20200611',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'uploader': 'chixa33',
+            'description': 'md5:5b067801318e33c2e6eea4ab90b1fdd3',
+        },
+        # working from local dev system
+        'skip': 'site blocks CI servers',
+    }
+    _MONTH_NAMES_KA = ['იანვარი', 'თებერვალი', 'მარტი', 'აპრილი', 'მაისი', 'ივნისი', 'ივლისი', 'აგვისტო', 'სექტემბერი', 'ოქტომბერი', 'ნოემბერი', 'დეკემბერი']
+
+    _quality = staticmethod(qualities(('SD', 'HD')))
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        title = (
+            self._og_search_title(webpage, default=None)
+            or clean_html(get_element_by_class('my_video_title', webpage))
+            or self._html_search_regex(r'<title\b[^>]*>([^<]+)</title\b', webpage, 'title'))
+
+        jwplayer_sources = self._parse_json(
+            self._search_regex(
+                r'''(?s)jwplayer\s*\(\s*['"]mvplayer['"]\s*\)\s*\.\s*setup\s*\(.*?\bsources\s*:\s*(\[.*?])\s*[,});]''', webpage, 'jwplayer sources', fatal=False)
+            or '',
+            video_id, transform_source=js_to_json, fatal=False)
+
+        formats = self._parse_jwplayer_formats(jwplayer_sources or [], video_id)
+        for f in formats or []:
+            f['preference'] = self._quality(f['format_id'])
+        self._sort_formats(formats)
+
+        description = (
+            self._og_search_description(webpage)
+            or get_element_by_id('long_desc_holder', webpage)
+            or self._html_search_meta('description', webpage))
+
+        uploader = self._search_regex(r'<a[^>]+class="mv_user_name"[^>]*>([^<]+)<', webpage, 'uploader', fatal=False)
+
+        upload_date = get_element_by_class('mv_vid_upl_date', webpage)
+        # as ka locale may not be present roll a local date conversion
+        upload_date = (unified_strdate(
+            # translate any ka month to an en one
+            re.sub('|'.join(self._MONTH_NAMES_KA),
+                   lambda m: MONTH_NAMES['en'][self._MONTH_NAMES_KA.index(m.group(0))],
+                   upload_date, re.I))
+            if upload_date else None)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': description,
+            'uploader': uploader,
+            'formats': formats,
+            'thumbnail': self._og_search_thumbnail(webpage),
+            'upload_date': upload_date,
+            'view_count': int_or_none(get_element_by_class('mv_vid_views', webpage)),
+            'like_count': int_or_none(get_element_by_id('likes_count', webpage)),
+            'dislike_count': int_or_none(get_element_by_id('dislikes_count', webpage)),
+        }

From 384f632e8a9b61e864a26678d85b2b39933b9bae Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 3 Feb 2023 21:10:07 +0000
Subject: [PATCH 585/860] [ITV] Overhaul ITV extractor (#30266)

* support ITVX URLs (thanks Vangelis66)
* support legacy ITV Hub URLs
* include extraction fix 4c57dd2 from sleaux-meaux 3 May 2021
* include extraction fix 6fbcc16, fix by staubichsauger & pukkandan
* work-around duration parsing pending fix to utils.parse_duration
* apply default vanilla UA for pages and media to avoid site blocking
* also detect and report `Episode not found` instead of generic 404
* rework ITVBTCCIE with geo-block detection, best effort geo-restriction handling, news article support
* fix tests
---
 youtube_dl/extractor/itv.py | 382 ++++++++++++++++++++++++++++--------
 1 file changed, 299 insertions(+), 83 deletions(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index e86c40b42..7026139ea 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -3,123 +3,266 @@ from __future__ import unicode_literals
 
 import json
 import re
+import sys
 
 from .common import InfoExtractor
 from .brightcove import BrightcoveNewIE
+from ..compat import (
+    compat_HTTPError,
+    compat_integer_types,
+    compat_kwargs,
+    compat_urlparse,
+)
 from ..utils import (
     clean_html,
     determine_ext,
+    error_to_compat_str,
     extract_attributes,
-    get_element_by_class,
-    JSON_LD_RE,
+    ExtractorError,
+    get_element_by_attribute,
+    int_or_none,
     merge_dicts,
     parse_duration,
+    parse_iso8601,
+    remove_start,
     smuggle_url,
+    strip_or_none,
+    traverse_obj,
     url_or_none,
+    urljoin,
 )
 
 
-class ITVIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?itv\.com/hub/[^/]+/(?P<id>[0-9a-zA-Z]+)'
-    _GEO_COUNTRIES = ['GB']
+class ITVBaseIE(InfoExtractor):
+
+    def _search_nextjs_data(self, webpage, video_id, **kw):
+        transform_source = kw.pop('transform_source', None)
+        fatal = kw.pop('fatal', True)
+        return self._parse_json(
+            self._search_regex(
+                r'''<script\b[^>]+\bid=('|")__NEXT_DATA__\1[^>]*>(?P<js>[^<]+)</script>''',
+                webpage, 'next.js data', group='js', fatal=fatal, **kw),
+            video_id, transform_source=transform_source, fatal=fatal)
+
+    def __handle_request_webpage_error(self, err, video_id=None, errnote=None, fatal=True):
+        if errnote is False:
+            return False
+        if errnote is None:
+            errnote = 'Unable to download webpage'
+
+        errmsg = '%s: %s' % (errnote, error_to_compat_str(err))
+        if fatal:
+            raise ExtractorError(errmsg, sys.exc_info()[2], cause=err, video_id=video_id)
+        else:
+            self._downloader.report_warning(errmsg)
+            return False
+
+    @staticmethod
+    def _vanilla_ua_header():
+        return {'User-agent': 'Mozilla/5.0'}
+
+    def _download_webpage_handle(self, url, video_id, *args, **kwargs):
+        # specialised to (a) use vanilla UA (b) detect geo-block
+        params = self._downloader.params
+        nkwargs = {}
+        if (
+                'user_agent' not in params
+                and not any(re.match(r'(?i)user-agent\s*:', h)
+                            for h in (params.get('headers') or []))
+                and 'User-agent' not in (kwargs.get('headers') or {})):
+
+            kwargs.setdefault('headers', {})
+            kwargs['headers'] = self._vanilla_ua_header()
+            nkwargs = kwargs
+        if kwargs.get('expected_status') is not None:
+            exp = kwargs['expected_status']
+            if isinstance(exp, compat_integer_types):
+                exp = [exp]
+            if isinstance(exp, (list, tuple)) and 403 not in exp:
+                kwargs['expected_status'] = [403]
+                kwargs['expected_status'].extend(exp)
+                nkwargs = kwargs
+        else:
+            kwargs['expected_status'] = 403
+            nkwargs = kwargs
+
+        if nkwargs:
+            kwargs = compat_kwargs(kwargs)
+
+        ret = super(ITVBaseIE, self)._download_webpage_handle(url, video_id, *args, **kwargs)
+        if ret is False:
+            return ret
+        webpage, urlh = ret
+
+        if urlh.getcode() == 403:
+            # geo-block error is like this, with an unnecessary 'Of':
+            # '{\n  "Message" : "Request Originated Outside Of Allowed Geographic Region",\
+            # \n  "TransactionId" : "oas-magni-475082-xbYF0W"\n}'
+            if '"Request Originated Outside Of Allowed Geographic Region"' in webpage:
+                self.raise_geo_restricted(countries=['GB'])
+            ret = self.__handle_request_webpage_error(
+                compat_HTTPError(urlh.geturl(), 403, 'HTTP Error 403: Forbidden', urlh.headers, urlh),
+                fatal=kwargs.get('fatal'))
+
+        return ret
+
+
+class ITVIE(ITVBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?itv\.com/(?:(?P<w>watch)|hub)/[^/]+/(?(w)[\w-]+/)(?P<id>\w+)'
+    _IE_DESC = 'ITVX'
     _TESTS = [{
+        'note': 'Hub URLs redirect to ITVX',
         'url': 'https://www.itv.com/hub/liar/2a4547a0012',
-        'info_dict': {
-            'id': '2a4547a0012',
-            'ext': 'mp4',
-            'title': 'Liar - Series 2 - Episode 6',
-            'description': 'md5:d0f91536569dec79ea184f0a44cca089',
-            'series': 'Liar',
-            'season_number': 2,
-            'episode_number': 6,
-        },
-        'params': {
-            # m3u8 download
-            'skip_download': True,
-        },
+        'only_matching': True,
     }, {
-        # unavailable via data-playlist-url
+        'note': 'Hub page unavailable via data-playlist-url (404 now)',
         'url': 'https://www.itv.com/hub/through-the-keyhole/2a2271a0033',
         'only_matching': True,
     }, {
-        # InvalidVodcrid
+        'note': 'Hub page with InvalidVodcrid (404 now)',
         'url': 'https://www.itv.com/hub/james-martins-saturday-morning/2a5159a0034',
         'only_matching': True,
     }, {
-        # ContentUnavailable
+        'note': 'Hub page with ContentUnavailable (404 now)',
         'url': 'https://www.itv.com/hub/whos-doing-the-dishes/2a2898a0024',
         'only_matching': True,
-    }]
+    }, {
+        'note': 'ITVX, or itvX, show',
+        'url': 'https://www.itv.com/watch/vera/1a7314/1a7314a0014',
+        'md5': 'bd0ad666b2c058fffe7d036785880064',
+        'info_dict': {
+            'id': '1a7314a0014',
+            'ext': 'mp4',
+            'title': 'Vera - Series 3 - Episode 4 - Prodigal Son',
+            'description': 'Vera and her team investigate the fatal stabbing of an ex-Met police officer outside a busy Newcastle nightclub - but there aren\'t many clues.',
+            'timestamp': 1653591600,
+            'upload_date': '20220526',
+            'uploader': 'ITVX',
+            'thumbnail': r're:https://\w+\.itv\.com/images/(?:\w+/)+\d+x\d+\?',
+            'duration': 5340.8,
+            'age_limit': 16,
+            'series': 'Vera',
+            'series_number': 3,
+            'episode': 'Prodigal Son',
+            'episode_number': 4,
+            'channel': 'ITV3',
+            'categories': list,
+        },
+        'params': {
+            # m3u8 download
+            # 'skip_download': True,
+        },
+        'skip': 'only available in UK',
+    }, {
+        'note': 'Latest ITV news bulletin: details change daily',
+        'url': 'https://www.itv.com/watch/news/varies-but-is-not-checked/6js5d0f',
+        'info_dict': {
+            'id': '6js5d0f',
+            'ext': 'mp4',
+            'title': r're:The latest ITV News headlines - \S.+',
+            'description': r'''re:.* today's top stories from the ITV News team.$''',
+            'timestamp': int,
+            'upload_date': r're:2\d\d\d(?:0[1-9]|1[0-2])(?:[012][1-9]|3[01])',
+            'uploader': 'ITVX',
+            'thumbnail': r're:https://images\.ctfassets\.net/(?:\w+/)+[\w.]+\.(?:jpg|png)',
+            'duration': float,
+            'age_limit': None,
+        },
+        'params': {
+            # variable download
+            # 'skip_download': True,
+        },
+        'skip': 'only available in UK',
+    }
+    ]
+
+    def _og_extract(self, webpage, require_title=False):
+        return {
+            'title': self._og_search_title(webpage, fatal=require_title),
+            'description': self._og_search_description(webpage, default=None),
+            'thumbnail': self._og_search_thumbnail(webpage, default=None),
+            'uploader': self._og_search_property('site_name', webpage, default=None),
+        }
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        webpage = self._download_webpage(url, video_id)
-        params = extract_attributes(self._search_regex(
-            r'(?s)(<[^>]+id="video"[^>]*>)', webpage, 'params'))
 
-        ios_playlist_url = params.get('data-video-playlist') or params['data-video-id']
-        hmac = params['data-video-hmac']
+        webpage = self._download_webpage(url, video_id)
+
+        # now quite different params!
+        params = extract_attributes(self._search_regex(
+            r'''(<[^>]+\b(?:class|data-testid)\s*=\s*("|')genie-container\2[^>]*>)''',
+            webpage, 'params'))
+
+        ios_playlist_url = traverse_obj(
+            params, 'data-video-id', 'data-video-playlist',
+            get_all=False, expected_type=url_or_none)
+
         headers = self.geo_verification_headers()
         headers.update({
             'Accept': 'application/vnd.itv.vod.playlist.v2+json',
             'Content-Type': 'application/json',
-            'hmac': hmac.upper(),
         })
         ios_playlist = self._download_json(
             ios_playlist_url, video_id, data=json.dumps({
                 'user': {
-                    'itvUserId': '',
                     'entitlements': [],
-                    'token': ''
                 },
                 'device': {
-                    'manufacturer': 'Safari',
-                    'model': '5',
+                    'manufacturer': 'Mobile Safari',
+                    'model': '5.1',
                     'os': {
-                        'name': 'Windows NT',
-                        'version': '6.1',
-                        'type': 'desktop'
+                        'name': 'iOS',
+                        'version': '5.0',
+                        'type': ' mobile'
                     }
                 },
                 'client': {
                     'version': '4.1',
-                    'id': 'browser'
+                    'id': 'browser',
+                    'supportsAdPods': True,
+                    'service': 'itv.x',
+                    'appversion': '2.43.28',
                 },
                 'variantAvailability': {
+                    'player': 'hls',
                     'featureset': {
                         'min': ['hls', 'aes', 'outband-webvtt'],
                         'max': ['hls', 'aes', 'outband-webvtt']
                     },
-                    'platformTag': 'dotcom'
+                    'platformTag': 'mobile'
                 }
             }).encode(), headers=headers)
         video_data = ios_playlist['Playlist']['Video']
-        ios_base_url = video_data.get('Base')
+        ios_base_url = traverse_obj(video_data, 'Base', expected_type=url_or_none)
+
+        media_url = (
+            (lambda u: url_or_none(urljoin(ios_base_url, u)))
+            if ios_base_url else url_or_none)
 
         formats = []
-        for media_file in (video_data.get('MediaFiles') or []):
-            href = media_file.get('Href')
+        for media_file in traverse_obj(video_data, 'MediaFiles', expected_type=list) or []:
+            href = traverse_obj(media_file, 'Href', expected_type=media_url)
             if not href:
                 continue
-            if ios_base_url:
-                href = ios_base_url + href
             ext = determine_ext(href)
             if ext == 'm3u8':
                 formats.extend(self._extract_m3u8_formats(
-                    href, video_id, 'mp4', entry_protocol='m3u8_native',
+                    href, video_id, 'mp4', entry_protocol='m3u8',
                     m3u8_id='hls', fatal=False))
+
             else:
                 formats.append({
                     'url': href,
                 })
         self._sort_formats(formats)
+        for f in formats:
+            f.setdefault('http_headers', {})
+            f['http_headers'].update(self._vanilla_ua_header())
 
         subtitles = {}
-        subs = video_data.get('Subtitles') or []
-        for sub in subs:
-            if not isinstance(sub, dict):
-                continue
-            href = url_or_none(sub.get('Href'))
+        for sub in traverse_obj(video_data, 'Subtitles', expected_type=list) or []:
+            href = traverse_obj(sub, 'Href', expected_type=url_or_none)
             if not href:
                 continue
             subtitles.setdefault('en', []).append({
@@ -127,59 +270,132 @@ class ITVIE(InfoExtractor):
                 'ext': determine_ext(href, 'vtt'),
             })
 
-        info = self._search_json_ld(webpage, video_id, default={})
-        if not info:
-            json_ld = self._parse_json(self._search_regex(
-                JSON_LD_RE, webpage, 'JSON-LD', '{}',
-                group='json_ld'), video_id, fatal=False)
-            if json_ld and json_ld.get('@type') == 'BreadcrumbList':
-                for ile in (json_ld.get('itemListElement:') or []):
-                    item = ile.get('item:') or {}
-                    if item.get('@type') == 'TVEpisode':
-                        item['@context'] = 'http://schema.org'
-                        info = self._json_ld(item, video_id, fatal=False) or {}
-                        break
+        next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default='{}')
+        video_data.update(traverse_obj(next_data, ('props', 'pageProps', ('title', 'episode')), expected_type=dict)[0] or {})
+        title = traverse_obj(video_data, 'headerTitle', 'episodeTitle')
+        info = self._og_extract(webpage, require_title=not title)
+        tn = info.pop('thumbnail', None)
+        if tn:
+            info['thumbnails'] = [{'url': tn}]
+
+        # num. episode title
+        num_ep_title = video_data.get('numberedEpisodeTitle')
+        if not num_ep_title:
+            num_ep_title = clean_html(get_element_by_attribute('data-testid', 'episode-hero-description-strong', webpage))
+            num_ep_title = num_ep_title and num_ep_title.rstrip(' -')
+        ep_title = strip_or_none(
+            video_data.get('episodeTitle')
+            or (num_ep_title.split('.', 1)[-1] if num_ep_title else None))
+        title = title or re.sub(r'\s+-\s+ITVX$', '', info['title'])
+        if ep_title and ep_title != title:
+            title = title + ' - ' + ep_title
+
+        def get_thumbnails():
+            tns = []
+            for w, x in (traverse_obj(video_data, ('imagePresets'), expected_type=dict) or {}).items():
+                if isinstance(x, dict):
+                    for y, z in x.items():
+                        tns.append({'id': w + '_' + y, 'url': z})
+            return tns or None
+
+        video_str = lambda *x: traverse_obj(
+            video_data, *x, get_all=False, expected_type=strip_or_none)
 
         return merge_dicts({
             'id': video_id,
-            'title': self._html_search_meta(['og:title', 'twitter:title'], webpage),
+            'title': title,
             'formats': formats,
             'subtitles': subtitles,
-            'duration': parse_duration(video_data.get('Duration')),
-            'description': clean_html(get_element_by_class('episode-info__synopsis', webpage)),
+            # parsing hh:mm:ss:nnn not yet patched
+            'duration': parse_duration(re.sub(r'(\d{2})(:)(\d{3}$)', r'\1.\3', video_data.get('Duration') or '')),
+            'description': video_str('synopsis'),
+            'timestamp': traverse_obj(video_data, 'broadcastDateTime', 'dateTime', expected_type=parse_iso8601),
+            'thumbnails': get_thumbnails(),
+            'series': video_str('showTitle', 'programmeTitle'),
+            'series_number': int_or_none(video_data.get('seriesNumber')),
+            'episode': ep_title,
+            'episode_number': int_or_none((num_ep_title or '').split('.')[0]),
+            'channel': video_str('channel'),
+            'categories': traverse_obj(video_data, ('categories', 'formatted'), expected_type=list),
+            'age_limit': {False: 16, True: 0}.get(video_data.get('isChildrenCategory')),
         }, info)
 
 
-class ITVBTCCIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?itv\.com/btcc/(?:[^/]+/)*(?P<id>[^/?#&]+)'
-    _TEST = {
-        'url': 'http://www.itv.com/btcc/races/btcc-2018-all-the-action-from-brands-hatch',
+class ITVBTCCIE(ITVBaseIE):
+    _VALID_URL = r'https?://(?:www\.)?itv\.com/(?!(?:watch|hub)/)(?:[^/]+/)+(?P<id>[^/?#&]+)'
+    _IE_DESC = 'ITV articles: News, British Touring Car Championship'
+    _TESTS = [{
+        'note': 'British Touring Car Championship',
+        'url': 'https://www.itv.com/btcc/articles/btcc-2018-all-the-action-from-brands-hatch',
         'info_dict': {
             'id': 'btcc-2018-all-the-action-from-brands-hatch',
             'title': 'BTCC 2018: All the action from Brands Hatch',
         },
         'playlist_mincount': 9,
-    }
-    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/1582188683001/HkiHLnNRx_default/index.html?videoId=%s'
+    }, {
+        'note': 'redirects to /btcc/articles/...',
+        'url': 'http://www.itv.com/btcc/races/btcc-2018-all-the-action-from-brands-hatch',
+        'only_matching': True,
+    }, {
+        'note': 'news article',
+        'url': 'https://www.itv.com/news/wales/2020-07-23/sean-fletcher-shows-off-wales-coastline-in-new-itv-series-as-british-tourists-opt-for-staycations',
+        'info_dict': {
+            'id': 'sean-fletcher-shows-off-wales-coastline-in-new-itv-series-as-british-tourists-opt-for-staycations',
+            'title': '''Sean Fletcher on why Wales' coastline should be your 'staycation' destination | ITV News''',
+        },
+        'playlist_mincount': 1,
+    }]
+
+    # should really be a class var of the BC IE
+    BRIGHTCOVE_URL_TEMPLATE = 'http://players.brightcove.net/%s/%s_default/index.html?videoId=%s'
+    BRIGHTCOVE_ACCOUNT = '1582188683001'
+    BRIGHTCOVE_PLAYER = 'HkiHLnNRx'
 
     def _real_extract(self, url):
         playlist_id = self._match_id(url)
 
-        webpage = self._download_webpage(url, playlist_id)
+        webpage, urlh = self._download_webpage_handle(url, playlist_id)
+        link = compat_urlparse.urlparse(urlh.geturl()).path.strip('/')
 
-        entries = [
-            self.url_result(
-                smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % video_id, {
-                    # ITV does not like some GB IP ranges, so here are some
-                    # IP blocks it accepts
-                    'geo_ip_blocks': [
-                        '193.113.0.0/16', '54.36.162.0/23', '159.65.16.0/21'
-                    ],
-                    'referrer': url,
-                }),
-                ie=BrightcoveNewIE.ie_key(), video_id=video_id)
-            for video_id in re.findall(r'data-video-id=["\'](\d+)', webpage)]
+        next_data = self._search_nextjs_data(webpage, playlist_id, fatal=False, default='{}')
+        path_prefix = compat_urlparse.urlparse(next_data.get('assetPrefix') or '').path.strip('/')
+        link = remove_start(link, path_prefix).strip('/')
+
+        content = traverse_obj(
+            next_data, ('props', 'pageProps', Ellipsis),
+            expected_type=lambda x: x if x['link'] == link else None,
+            get_all=False, default={})
+        content = traverse_obj(
+            content, ('body', 'content', Ellipsis, 'data'),
+            expected_type=lambda x: x if x.get('name') == 'Brightcove' or x.get('type') == 'Brightcove' else None)
+
+        contraband = {
+            # ITV does not like some GB IP ranges, so here are some
+            # IP blocks it accepts
+            'geo_ip_blocks': [
+                '193.113.0.0/16', '54.36.162.0/23', '159.65.16.0/21'
+            ],
+            'referrer': urlh.geturl(),
+        }
+
+        def entries():
+
+            for data in content or []:
+                video_id = data.get('id')
+                if not video_id:
+                    continue
+                account = data.get('accountId') or self.BRIGHTCOVE_ACCOUNT
+                player = data.get('playerId') or self.BRIGHTCOVE_PLAYER
+                yield self.url_result(
+                    smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % (account, player, video_id), contraband),
+                    ie=BrightcoveNewIE.ie_key(), video_id=video_id)
+
+            # obsolete ?
+            for video_id in re.findall(r'''data-video-id=["'](\d+)''', webpage):
+                yield self.url_result(
+                    smuggle_url(self.BRIGHTCOVE_URL_TEMPLATE % (self.BRIGHTCOVE_ACCOUNT, self.BRIGHTCOVE_PLAYER, video_id), contraband),
+                    ie=BrightcoveNewIE.ie_key(), video_id=video_id)
 
         title = self._og_search_title(webpage, fatal=False)
 
-        return self.playlist_result(entries, playlist_id, title)
+        return self.playlist_result(entries(), playlist_id, title)

From d947ffe8e385a541f44c6125b4cbc269de6055a4 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 4 Feb 2023 00:19:48 +0000
Subject: [PATCH 586/860] [IGN] Overhaul extractor to avoid URL redirection
 loop

Consequently/also:
* centralise video data extraction
* detect 404 and 503 expected errors
* handle the test video in IGNVideo
* handle two additional page formats for the tests in IGNArticle
---
 youtube_dl/extractor/ign.py | 347 ++++++++++++++++++++++++++----------
 1 file changed, 252 insertions(+), 95 deletions(-)

diff --git a/youtube_dl/extractor/ign.py b/youtube_dl/extractor/ign.py
index 0d9f50ed2..c7daa30e5 100644
--- a/youtube_dl/extractor/ign.py
+++ b/youtube_dl/extractor/ign.py
@@ -1,19 +1,29 @@
+# coding: utf-8
+
 from __future__ import unicode_literals
 
 import re
 
 from .common import InfoExtractor
 from ..compat import (
+    compat_filter as filter,
+    compat_HTTPError,
     compat_parse_qs,
-    compat_urllib_parse_urlparse,
+    compat_urlparse,
 )
 from ..utils import (
-    HEADRequest,
     determine_ext,
+    error_to_compat_str,
+    extract_attributes,
+    ExtractorError,
     int_or_none,
+    merge_dicts,
+    orderedSet,
     parse_iso8601,
     strip_or_none,
-    try_get,
+    traverse_obj,
+    url_or_none,
+    urljoin,
 )
 
 
@@ -22,14 +32,102 @@ class IGNBaseIE(InfoExtractor):
         return self._download_json(
             'http://apis.ign.com/{0}/v3/{0}s/slug/{1}'.format(self._PAGE_TYPE, slug), slug)
 
+    def _checked_call_api(self, slug):
+        try:
+            return self._call_api(slug)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError) and e.cause.code == 404:
+                e.cause.args = e.cause.args or [
+                    e.cause.geturl(), e.cause.getcode(), e.cause.reason]
+                raise ExtractorError(
+                    'Content not found: expired?', cause=e.cause,
+                    expected=True)
+            raise
+
+    def _extract_video_info(self, video, fatal=True):
+        video_id = video['videoId']
+
+        formats = []
+        refs = traverse_obj(video, 'refs', expected_type=dict) or {}
+
+        m3u8_url = url_or_none(refs.get('m3uUrl'))
+        if m3u8_url:
+            formats.extend(self._extract_m3u8_formats(
+                m3u8_url, video_id, 'mp4', 'm3u8_native',
+                m3u8_id='hls', fatal=False))
+
+        f4m_url = url_or_none(refs.get('f4mUrl'))
+        if f4m_url:
+            formats.extend(self._extract_f4m_formats(
+                f4m_url, video_id, f4m_id='hds', fatal=False))
+
+        for asset in (video.get('assets') or []):
+            asset_url = url_or_none(asset.get('url'))
+            if not asset_url:
+                continue
+            formats.append({
+                'url': asset_url,
+                'tbr': int_or_none(asset.get('bitrate'), 1000),
+                'fps': int_or_none(asset.get('frame_rate')),
+                'height': int_or_none(asset.get('height')),
+                'width': int_or_none(asset.get('width')),
+            })
+
+        mezzanine_url = traverse_obj(
+            video, ('system', 'mezzanineUrl'), expected_type=url_or_none)
+        if mezzanine_url:
+            formats.append({
+                'ext': determine_ext(mezzanine_url, 'mp4'),
+                'format_id': 'mezzanine',
+                'preference': 1,
+                'url': mezzanine_url,
+            })
+
+        if formats or fatal:
+            self._sort_formats(formats)
+        else:
+            return
+
+        thumbnails = traverse_obj(
+            video, ('thumbnails', Ellipsis, {'url': 'url'}), expected_type=url_or_none)
+        tags = traverse_obj(
+            video, ('tags', Ellipsis, 'displayName'),
+            expected_type=lambda x: x.strip() or None)
+
+        metadata = traverse_obj(video, 'metadata', expected_type=dict) or {}
+        title = traverse_obj(
+            metadata, 'longTitle', 'title', 'name',
+            expected_type=lambda x: x.strip() or None)
+
+        return {
+            'id': video_id,
+            'title': title,
+            'description': strip_or_none(metadata.get('description')),
+            'timestamp': parse_iso8601(metadata.get('publishDate')),
+            'duration': int_or_none(metadata.get('duration')),
+            'thumbnails': thumbnails,
+            'formats': formats,
+            'tags': tags,
+        }
+
+    # yt-dlp shim
+    @classmethod
+    def _extract_from_webpage(cls, url, webpage):
+        for embed_url in orderedSet(
+                cls._extract_embed_urls(url, webpage) or [], lazy=True):
+            yield cls.url_result(embed_url, None if cls._VALID_URL is False else cls)
+
 
 class IGNIE(IGNBaseIE):
     """
     Extractor for some of the IGN sites, like www.ign.com, es.ign.com de.ign.com.
     Some videos of it.ign.com are also supported
     """
-
-    _VALID_URL = r'https?://(?:.+?\.ign|www\.pcmag)\.com/videos/(?:\d{4}/\d{2}/\d{2}/)?(?P<id>[^/?&#]+)'
+    _VIDEO_PATH_RE = r'/(?:\d{4}/\d{2}/\d{2}/)?(?P<id>.+?)'
+    _PLAYLIST_PATH_RE = r'(?:/?\?(?P<filt>[^&#]+))?'
+    _VALID_URL = (
+        r'https?://(?:.+?\.ign|www\.pcmag)\.com/videos(?:%s)'
+        % '|'.join((_VIDEO_PATH_RE + r'(?:[/?&#]|$)', _PLAYLIST_PATH_RE)))
     IE_NAME = 'ign.com'
     _PAGE_TYPE = 'video'
 
@@ -44,7 +142,10 @@ class IGNIE(IGNBaseIE):
             'timestamp': 1370440800,
             'upload_date': '20130605',
             'tags': 'count:9',
-        }
+        },
+        'params': {
+            'nocheckcertificate': True,
+        },
     }, {
         'url': 'http://www.pcmag.com/videos/2015/01/06/010615-whats-new-now-is-gogo-snooping-on-your-data',
         'md5': 'f1581a6fe8c5121be5b807684aeac3f6',
@@ -56,86 +157,51 @@ class IGNIE(IGNBaseIE):
             'timestamp': 1420571160,
             'upload_date': '20150106',
             'tags': 'count:4',
-        }
+        },
+        'skip': '404 Not Found',
     }, {
         'url': 'https://www.ign.com/videos/is-a-resident-evil-4-remake-on-the-way-ign-daily-fix',
         'only_matching': True,
     }]
 
+    @classmethod
+    def _extract_embed_urls(cls, url, webpage):
+        grids = re.findall(
+            r'''(?s)<section\b[^>]+\bclass\s*=\s*['"](?:[\w-]+\s+)*?content-feed-grid(?!\B|-)[^>]+>(.+?)</section[^>]*>''',
+            webpage)
+        return filter(None,
+                      (urljoin(url, m.group('path')) for m in re.finditer(
+                          r'''<a\b[^>]+\bhref\s*=\s*('|")(?P<path>/videos%s)\1'''
+                          % cls._VIDEO_PATH_RE, grids[0] if grids else '')))
+
     def _real_extract(self, url):
+        m = re.match(self._VALID_URL, url)
+        display_id = m.group('id')
+        if display_id:
+            return self._extract_video(url, display_id)
+        display_id = m.group('filt') or 'all'
+        return self._extract_playlist(url, display_id)
+
+    def _extract_playlist(self, url, display_id):
+        webpage = self._download_webpage(url, display_id)
+
+        return self.playlist_result(
+            (self.url_result(u, ie=self.ie_key())
+             for u in self._extract_embed_urls(url, webpage)),
+            playlist_id=display_id)
+
+    def _extract_video(self, url, display_id):
         display_id = self._match_id(url)
-        video = self._call_api(display_id)
-        video_id = video['videoId']
-        metadata = video['metadata']
-        title = metadata.get('longTitle') or metadata.get('title') or metadata['name']
+        video = self._checked_call_api(display_id)
 
-        formats = []
-        refs = video.get('refs') or {}
+        info = self._extract_video_info(video)
 
-        m3u8_url = refs.get('m3uUrl')
-        if m3u8_url:
-            formats.extend(self._extract_m3u8_formats(
-                m3u8_url, video_id, 'mp4', 'm3u8_native',
-                m3u8_id='hls', fatal=False))
-
-        f4m_url = refs.get('f4mUrl')
-        if f4m_url:
-            formats.extend(self._extract_f4m_formats(
-                f4m_url, video_id, f4m_id='hds', fatal=False))
-
-        for asset in (video.get('assets') or []):
-            asset_url = asset.get('url')
-            if not asset_url:
-                continue
-            formats.append({
-                'url': asset_url,
-                'tbr': int_or_none(asset.get('bitrate'), 1000),
-                'fps': int_or_none(asset.get('frame_rate')),
-                'height': int_or_none(asset.get('height')),
-                'width': int_or_none(asset.get('width')),
-            })
-
-        mezzanine_url = try_get(video, lambda x: x['system']['mezzanineUrl'])
-        if mezzanine_url:
-            formats.append({
-                'ext': determine_ext(mezzanine_url, 'mp4'),
-                'format_id': 'mezzanine',
-                'preference': 1,
-                'url': mezzanine_url,
-            })
-
-        self._sort_formats(formats)
-
-        thumbnails = []
-        for thumbnail in (video.get('thumbnails') or []):
-            thumbnail_url = thumbnail.get('url')
-            if not thumbnail_url:
-                continue
-            thumbnails.append({
-                'url': thumbnail_url,
-            })
-
-        tags = []
-        for tag in (video.get('tags') or []):
-            display_name = tag.get('displayName')
-            if not display_name:
-                continue
-            tags.append(display_name)
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': strip_or_none(metadata.get('description')),
-            'timestamp': parse_iso8601(metadata.get('publishDate')),
-            'duration': int_or_none(metadata.get('duration')),
+        return merge_dicts({
             'display_id': display_id,
-            'thumbnails': thumbnails,
-            'formats': formats,
-            'tags': tags,
-        }
+        }, info)
 
 
-class IGNVideoIE(InfoExtractor):
+class IGNVideoIE(IGNBaseIE):
     _VALID_URL = r'https?://.+?\.ign\.com/(?:[a-z]{2}/)?[^/]+/(?P<id>\d+)/(?:video|trailer)/'
     _TESTS = [{
         'url': 'http://me.ign.com/en/videos/112203/video/how-hitman-aims-to-be-different-than-every-other-s',
@@ -147,7 +213,8 @@ class IGNVideoIE(InfoExtractor):
             'description': 'Taking out assassination targets in Hitman has never been more stylish.',
             'timestamp': 1444665600,
             'upload_date': '20151012',
-        }
+        },
+        'expected_warnings': ['HTTP Error 400: Bad Request'],
     }, {
         'url': 'http://me.ign.com/ar/angry-birds-2/106533/video/lrd-ldyy-lwl-lfylm-angry-birds',
         'only_matching': True,
@@ -167,22 +234,38 @@ class IGNVideoIE(InfoExtractor):
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
-        req = HEADRequest(url.rsplit('/', 1)[0] + '/embed')
-        url = self._request_webpage(req, video_id).geturl()
+        parsed_url = compat_urlparse.urlparse(url)
+        embed_url = compat_urlparse.urlunparse(
+            parsed_url._replace(path=parsed_url.path.rsplit('/', 1)[0] + '/embed'))
+
+        webpage, urlh = self._download_webpage_handle(embed_url, video_id)
+        new_url = urlh.geturl()
         ign_url = compat_parse_qs(
-            compat_urllib_parse_urlparse(url).query).get('url', [None])[0]
+            compat_urlparse.urlparse(new_url).query).get('url', [None])[-1]
         if ign_url:
             return self.url_result(ign_url, IGNIE.ie_key())
-        return self.url_result(url)
+        video = self._search_regex(r'(<div\b[^>]+\bdata-video-id\s*=\s*[^>]+>)', webpage, 'video element', fatal=False)
+        if not video:
+            if new_url == url:
+                raise ExtractorError('Redirect loop: ' + url)
+            return self.url_result(new_url)
+        video = extract_attributes(video)
+        video_data = video.get('data-settings') or '{}'
+        video_data = self._parse_json(video_data, video_id)['video']
+        info = self._extract_video_info(video_data)
+
+        return merge_dicts({
+            'display_id': video_id,
+        }, info)
 
 
 class IGNArticleIE(IGNBaseIE):
-    _VALID_URL = r'https?://.+?\.ign\.com/(?:articles(?:/\d{4}/\d{2}/\d{2})?|(?:[a-z]{2}/)?feature/\d+)/(?P<id>[^/?&#]+)'
+    _VALID_URL = r'https?://.+?\.ign\.com/(?:articles(?:/\d{4}/\d{2}/\d{2})?|(?:[a-z]{2}/)?(?:[\w-]+/)*?feature/\d+)/(?P<id>[^/?&#]+)'
     _PAGE_TYPE = 'article'
     _TESTS = [{
         'url': 'http://me.ign.com/en/feature/15775/100-little-things-in-gta-5-that-will-blow-your-mind',
         'info_dict': {
-            'id': '524497489e4e8ff5848ece34',
+            'id': '72113',
             'title': '100 Little Things in GTA 5 That Will Blow Your Mind',
         },
         'playlist': [
@@ -190,7 +273,7 @@ class IGNArticleIE(IGNBaseIE):
                 'info_dict': {
                     'id': '5ebbd138523268b93c9141af17bec937',
                     'ext': 'mp4',
-                    'title': 'GTA 5 Video Review',
+                    'title': 'Grand Theft Auto V Video Review',
                     'description': 'Rockstar drops the mic on this generation of games. Watch our review of the masterly Grand Theft Auto V.',
                     'timestamp': 1379339880,
                     'upload_date': '20130916',
@@ -200,7 +283,7 @@ class IGNArticleIE(IGNBaseIE):
                 'info_dict': {
                     'id': '638672ee848ae4ff108df2a296418ee2',
                     'ext': 'mp4',
-                    'title': '26 Twisted Moments from GTA 5 in Slow Motion',
+                    'title': 'GTA 5 In Slow Motion',
                     'description': 'The twisted beauty of GTA 5 in stunning slow motion.',
                     'timestamp': 1386878820,
                     'upload_date': '20131212',
@@ -208,16 +291,17 @@ class IGNArticleIE(IGNBaseIE):
             },
         ],
         'params': {
-            'playlist_items': '2-3',
             'skip_download': True,
         },
+        'expected_warnings': ['Backend fetch failed'],
     }, {
         'url': 'http://www.ign.com/articles/2014/08/15/rewind-theater-wild-trailer-gamescom-2014?watch',
         'info_dict': {
             'id': '53ee806780a81ec46e0790f8',
             'title': 'Rewind Theater - Wild Trailer Gamescom 2014',
         },
-        'playlist_count': 2,
+        'playlist_count': 1,
+        'expected_warnings': ['Backend fetch failed'],
     }, {
         # videoId pattern
         'url': 'http://www.ign.com/articles/2017/06/08/new-ducktales-short-donalds-birthday-doesnt-go-as-planned',
@@ -240,18 +324,91 @@ class IGNArticleIE(IGNBaseIE):
         'only_matching': True,
     }]
 
+    def _checked_call_api(self, slug):
+        try:
+            return self._call_api(slug)
+        except ExtractorError as e:
+            if isinstance(e.cause, compat_HTTPError):
+                e.cause.args = e.cause.args or [
+                    e.cause.geturl(), e.cause.getcode(), e.cause.reason]
+                if e.cause.code == 404:
+                    raise ExtractorError(
+                        'Content not found: expired?', cause=e.cause,
+                        expected=True)
+                elif e.cause.code == 503:
+                    self.report_warning(error_to_compat_str(e.cause))
+                    return
+            raise
+
+    def _search_nextjs_data(self, webpage, video_id, **kw):
+        return self._parse_json(
+            self._search_regex(
+                r'(?s)<script[^>]+id=[\'"]__NEXT_DATA__[\'"][^>]*>([^<]+)</script>',
+                webpage, 'next.js data', **kw),
+            video_id, **kw)
+
     def _real_extract(self, url):
         display_id = self._match_id(url)
-        article = self._call_api(display_id)
+        article = self._checked_call_api(display_id)
 
-        def entries():
-            media_url = try_get(article, lambda x: x['mediaRelations'][0]['media']['metadata']['url'])
-            if media_url:
-                yield self.url_result(media_url, IGNIE.ie_key())
-            for content in (article.get('content') or []):
-                for video_url in re.findall(r'(?:\[(?:ignvideo\s+url|youtube\s+clip_id)|<iframe[^>]+src)="([^"]+)"', content):
-                    yield self.url_result(video_url)
+        if article:
+            # obsolete ?
+            def entries():
+                media_url = traverse_obj(
+                    article, ('mediaRelations', 0, 'media', 'metadata', 'url'),
+                    expected_type=url_or_none)
+                if media_url:
+                    yield self.url_result(media_url, IGNIE.ie_key())
+                for content in (article.get('content') or []):
+                    for video_url in re.findall(r'(?:\[(?:ignvideo\s+url|youtube\s+clip_id)|<iframe[^>]+src)="([^"]+)"', content):
+                        if url_or_none(video_url):
+                            yield self.url_result(video_url)
+
+            return self.playlist_result(
+                entries(), article.get('articleId'),
+                traverse_obj(
+                    article, ('metadata', 'headline'),
+                    expected_type=lambda x: x.strip() or None))
+
+        webpage = self._download_webpage(url, display_id)
+
+        playlist_id = self._html_search_meta('dable:item_id', webpage, default=None)
+        if playlist_id:
+
+            def entries():
+                for m in re.finditer(
+                        r'''(?s)<object\b[^>]+\bclass\s*=\s*("|')ign-videoplayer\1[^>]*>(?P<params>.+?)</object''',
+                        webpage):
+                    flashvars = self._search_regex(
+                        r'''(<param\b[^>]+\bname\s*=\s*("|')flashvars\2[^>]*>)''',
+                        m.group('params'), 'flashvars', default='')
+                    flashvars = compat_parse_qs(extract_attributes(flashvars).get('value') or '')
+                    v_url = url_or_none((flashvars.get('url') or [None])[-1])
+                    if v_url:
+                        yield self.url_result(v_url)
+        else:
+            playlist_id = self._search_regex(
+                r'''\bdata-post-id\s*=\s*("|')(?P<id>[\da-f]+)\1''',
+                webpage, 'id', group='id', default=None)
+
+            nextjs_data = self._search_nextjs_data(webpage, display_id)
+
+            def entries():
+                for player in traverse_obj(
+                        nextjs_data,
+                        ('props', 'apolloState', 'ROOT_QUERY', lambda k, _: k.startswith('videoPlayerProps('), '__ref')):
+                    # skip promo links (which may not always be served, eg GH CI servers)
+                    if traverse_obj(nextjs_data,
+                                    ('props', 'apolloState', player.replace('PlayerProps', 'ModernContent')),
+                                    expected_type=dict):
+                        continue
+                    video = traverse_obj(nextjs_data, ('props', 'apolloState', player), expected_type=dict) or {}
+                    info = self._extract_video_info(video, fatal=False)
+                    if info:
+                        yield merge_dicts({
+                            'display_id': display_id,
+                        }, info)
 
         return self.playlist_result(
-            entries(), article.get('articleId'),
-            strip_or_none(try_get(article, lambda x: x['metadata']['headline'])))
+            entries(), playlist_id or display_id,
+            re.sub(r'\s+-\s+IGN\s*$', '', self._og_search_title(webpage, default='')) or None)

From cd987e6fca336cf6570b4938442c23cd0bdf7256 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 4 Feb 2023 01:53:47 +0000
Subject: [PATCH 587/860] [jsinterp] Nits

---
 youtube_dl/jsinterp.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 1e7b342ac..60fa2b1b9 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -201,7 +201,7 @@ class JSInterpreter(object):
         def __init__(self, msg, *args, **kwargs):
             expr = kwargs.pop('expr', None)
             if expr is not None:
-                msg = '{0} in: {1!r}'.format(msg.rstrip(), expr[:100])
+                msg = '{0} in: {1!r:.100}'.format(msg.rstrip(), expr)
             super(JSInterpreter.Exception, self).__init__(msg, *args, **kwargs)
 
     class JS_RegExp(object):
@@ -699,7 +699,7 @@ class JSInterpreter(object):
                 """ assert, but without risk of getting optimized out """
                 if not cndn:
                     memb = member
-                    raise self.Exception('{member} {msg}'.format(**locals()), expr=expr)
+                    raise self.Exception('{memb} {msg}'.format(**locals()), expr=expr)
 
             def eval_method():
                 if (variable, member) == ('console', 'debug'):

From f2f90887ca7a452dfafa7ca221fe981a4ec56707 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 4 Feb 2023 00:21:35 +0000
Subject: [PATCH 588/860] [Vimeo] Fix `Unable to extract info section` redux *
 as reported in yt-dlp/yt-dlp#6149 * also allow newline in target JSON object

---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 853b38402..14f8dd034 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -663,7 +663,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
 
         if '//player.vimeo.com/video/' in url:
             config = self._parse_json(self._search_regex(
-                r'\b(?:playerC|c)onfig\s*=\s*({.+?})\s*;', webpage, 'info section'), video_id)
+                r'(?s)\b(?:playerC|c)onfig\s*=\s*({.+?})\s*[;\n]', webpage, 'info section'), video_id)
             if config.get('view') == 4:
                 config = self._verify_player_video_password(
                     redirect_url, video_id, headers)

From e19ec5232216fd801ded88728df5b50bfb05c1cc Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 11 Feb 2023 03:25:14 +0000
Subject: [PATCH 589/860] [Vimeo] Support /user{video_id}/{slug} URL format

---
 youtube_dl/extractor/vimeo.py | 55 +++++++++++++++++++++--------------
 1 file changed, 33 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 14f8dd034..7f2731d83 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -261,27 +261,33 @@ class VimeoIE(VimeoBaseInfoExtractor):
 
     # _VALID_URL matches Vimeo URLs
     _VALID_URL = r'''(?x)
-                    https?://
-                        (?:
-                            (?:
-                                www|
-                                player
-                            )
-                            \.
-                        )?
-                        vimeo(?:pro)?\.com/
-                        (?!(?:channels|album|showcase)/[^/?#]+/?(?:$|[?#])|[^/]+/review/|ondemand/)
-                        (?:.*?/)??
-                        (?:
-                            (?:
-                                play_redirect_hls|
-                                moogaloop\.swf)\?clip_id=
-                            )?
-                        (?:videos?/)?
-                        (?P<id>[0-9]+)
-                        (?:/(?P<unlisted_hash>[\da-f]{10}))?
-                        /?(?:[?&].*)?(?:[#].*)?$
-                    '''
+                     https?://
+                         (?:
+                             (?:
+                                 www|
+                                 player
+                             )
+                             \.
+                         )?
+                         vimeo(?:pro)?\.com/
+                         (?:
+                             (?P<u>user)|
+                             (?!(?:channels|album|showcase)/[^/?#]+/?(?:$|[?#])|[^/]+/review/|ondemand/)
+                             (?:.*?/)??
+                             (?P<q>
+                                 (?:
+                                     play_redirect_hls|
+                                     moogaloop\.swf)\?clip_id=
+                             )?
+                             (?:videos?/)?
+                         )
+                         (?P<id>[0-9]+)
+                         (?(u)
+                             /(?!videos|likes)[^/?#]+/?|
+                             (?(q)|/(?P<unlisted_hash>[\da-f]{10}))?
+                         )
+                         (?:(?(q)[&]|(?(u)|/?)[?]).*?)?(?:[#].*)?$
+                 '''
     IE_NAME = 'vimeo'
     _TESTS = [
         {
@@ -539,7 +545,12 @@ class VimeoIE(VimeoBaseInfoExtractor):
             'params': {
                 'skip_download': True,
             },
-        }
+        },
+        {
+            # user playlist alias -> https://vimeo.com/258705797
+            'url': 'https://vimeo.com/user26785108/newspiritualguide',
+            'only_matching': True,
+        },
         # https://gettingthingsdone.com/workflowmap/
         # vimeo embed with check-password page protected by Referer header
     ]

From 58988c1421b88875a33015b08e4d2ada43021e09 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 31 Jan 2022 04:28:54 +0000
Subject: [PATCH 590/860] [YouTube] Bypass age-gating for certain restricted
 videos

* Use TVHTML5_SIMPLY_EMBEDDED_PLAYER client

* Also add and fix tests

* Introduce and use new utility function `update_url()`
---
 youtube_dl/extractor/youtube.py | 202 +++++++++++++++++++++++++-------
 youtube_dl/utils.py             |  11 ++
 2 files changed, 168 insertions(+), 45 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 28fdb086a..65428528d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -42,6 +42,7 @@ from ..utils import (
     unescapeHTML,
     unified_strdate,
     unsmuggle_url,
+    update_url,
     update_url_query,
     url_or_none,
     urlencode_postdata,
@@ -286,15 +287,18 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
     _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
 
-    def _call_api(self, ep, query, video_id, fatal=True):
+    def _call_api(self, ep, query, video_id, fatal=True, headers=None):
         data = self._DEFAULT_API_DATA.copy()
         data.update(query)
+        real_headers = {'content-type': 'application/json'}
+        if headers:
+            real_headers.update(headers)
 
         return self._download_json(
             'https://www.youtube.com/youtubei/v1/%s' % ep, video_id=video_id,
             note='Downloading API JSON', errnote='Unable to download API page',
             data=json.dumps(data).encode('utf8'), fatal=fatal,
-            headers={'content-type': 'application/json'},
+            headers=real_headers,
             query={'key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'})
 
     def _extract_yt_initial_data(self, video_id, webpage):
@@ -515,6 +519,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'Philipp Hagemeister',
                 'uploader_id': 'phihag',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
+                'channel': 'Philipp Hagemeister',
                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
                 'upload_date': '20121002',
@@ -524,10 +529,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'duration': 10,
                 'view_count': int,
                 'like_count': int,
-                'dislike_count': int,
+                'thumbnail': 'https://i.ytimg.com/vi/BaW_jenozKc/maxresdefault.jpg',
                 'start_time': 1,
                 'end_time': 9,
-            }
+            },
         },
         {
             'url': '//www.YouTube.com/watch?v=yZIXLfi8CZQ',
@@ -562,7 +567,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'duration': 10,
                 'view_count': int,
                 'like_count': int,
-                'dislike_count': int,
             },
             'params': {
                 'skip_download': True,
@@ -621,8 +625,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms \r\n\r\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
             }
         },
-        # Normal age-gate video (No vevo, embed allowed), available via embed page
+        # Age-gated videos
         {
+            'note': 'Age-gated video (No vevo, embed allowed)',
             'url': 'https://youtube.com/watch?v=HtVdAasjOgU',
             'info_dict': {
                 'id': 'HtVdAasjOgU',
@@ -631,17 +636,97 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
                 'duration': 142,
                 'uploader': 'The Witcher',
-                'uploader_id': 'WitcherGame',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
                 'upload_date': '20140605',
+                'thumbnail': 'https://i.ytimg.com/vi/HtVdAasjOgU/maxresdefault.jpg',
                 'age_limit': 18,
+                'categories': ['Gaming'],
+                'tags': 'count:17',
+                'channel': 'The Witcher',
+                'channel_url': 'https://www.youtube.com/channel/UCzybXLxv08IApdjdN0mJhEg',
+                'channel_id': 'UCzybXLxv08IApdjdN0mJhEg',
+                'view_count': int,
+                'like_count': int,
             },
         },
         {
-            # Age-gated video only available with authentication (unavailable
-            # via embed page workaround)
+            'note': 'Age-gated video with embed allowed in public site',
+            'url': 'https://youtube.com/watch?v=HsUATh_Nc2U',
+            'info_dict': {
+                'id': 'HsUATh_Nc2U',
+                'ext': 'mp4',
+                'title': 'Godzilla 2 (Official Video)',
+                'description': 'md5:bf77e03fcae5529475e500129b05668a',
+                'duration': 177,
+                'uploader': 'FlyingKitty',
+                'upload_date': '20200408',
+                'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
+                'age_limit': 18,
+                'categories': ['Entertainment'],
+                'tags': ['Flyingkitty', 'godzilla 2'],
+                'channel': 'FlyingKitty',
+                'channel_url': 'https://www.youtube.com/channel/UCYQT13AtrJC0gsM1far_zJg',
+                'channel_id': 'UCYQT13AtrJC0gsM1far_zJg',
+                'view_count': int,
+                'like_count': int,
+            },
+        },
+        {
+            'note': 'Age-gated video embedable only with clientScreen=EMBED',
+            'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
+            'info_dict': {
+                'id': 'Tq92D6wQ1mg',
+                'ext': 'mp4',
+                'title': '[MMD] Adios - EVERGLOW [+Motion DL]',
+                'description': 'md5:17eccca93a786d51bc67646756894066',
+                'duration': 106,
+                'uploader': 'Projekt Melody',
+                'upload_date': '20191227',
+                'age_limit': 18,
+                'thumbnail': 'https://i.ytimg.com/vi/Tq92D6wQ1mg/sddefault.jpg',
+                'tags': ['mmd', 'dance', 'mikumikudance', 'kpop', 'vtuber'],
+                'categories': ['Entertainment'],
+                'channel': 'Projekt Melody',
+                'channel_url': 'https://www.youtube.com/channel/UC1yoRdFoFJaCY-AGfD9W0wQ',
+                'channel_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
+                'view_count': int,
+                'like_count': int,
+            },
+        },
+        {
+            'note': 'Non-Age-gated non-embeddable video',
+            'url': 'https://youtube.com/watch?v=MeJVWBSsPAY',
+            'info_dict': {
+                'id': 'MeJVWBSsPAY',
+                'ext': 'mp4',
+                'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
+                'description': 'Fan Video. Music & Lyrics by OOMPH!.',
+                'duration': 210,
+                'uploader': 'Herr Lurik',
+                'uploader_id': 'st3in234',
+                'upload_date': '20130730',
+                'uploader_url': 'http://www.youtube.com/user/st3in234',
+                'age_limit': 0,
+                'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/hqdefault.jpg',
+                'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
+                'categories': ['Music'],
+                'channel': 'Herr Lurik',
+                'channel_url': 'https://www.youtube.com/channel/UCdR3RSDPqub28LjZx0v9-aA',
+                'channel_id': 'UCdR3RSDPqub28LjZx0v9-aA',
+                'artist': 'OOMPH!',
+                'view_count': int,
+                'like_count': int,
+            },
+        },
+        {
+            'note': 'Non-bypassable age-gated video',
+            'url': 'https://youtube.com/watch?v=Cr381pDsSsA',
+            'only_matching': True,
+        },
+        {
+            'note': 'Age-gated video only available with authentication (not via embed workaround)',
             'url': 'XgnwCQzjau8',
             'only_matching': True,
+            'skip': '''This video has been removed for violating YouTube's Community Guidelines''',
         },
         # video_info is None (https://github.com/ytdl-org/youtube-dl/issues/4421)
         # YouTube Red ad is not captured for creator
@@ -670,17 +755,23 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'info_dict': {
                 'id': 'lqQg6PlCWgI',
                 'ext': 'mp4',
+                'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
+                'description': r're:(?s)(?:.+\s)?HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games\s*',
                 'duration': 6085,
                 'upload_date': '20150827',
                 'uploader_id': 'olympic',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
-                'description': 'HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games',
-                'uploader': 'Olympic',
-                'title': 'Hockey - Women -  GER-AUS - London 2012 Olympic Games',
+                'uploader': r're:Olympics?',
+                'age_limit': 0,
+                'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
+                'categories': ['Sports'],
+                'tags': ['Hockey', '2012-07-31', '31 July 2012', 'Riverbank Arena', 'Session', 'Olympics', 'Olympic Games', 'London 2012', '2012 Summer Olympics', 'Summer Games'],
+                'channel': 'Olympics',
+                'channel_url': 'https://www.youtube.com/channel/UCTl3QQTvqHFjurroKxexy2Q',
+                'channel_id': 'UCTl3QQTvqHFjurroKxexy2Q',
+                'view_count': int,
+                'like_count': int,
             },
-            'params': {
-                'skip_download': 'requires avconv',
-            }
         },
         # Non-square pixels
         {
@@ -1683,27 +1774,52 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             player_response = self._call_api(
                 'player', {'videoId': video_id}, video_id)
 
-        playability_status = player_response.get('playabilityStatus') or {}
-        if playability_status.get('reason') == 'Sign in to confirm your age':
-            video_info = self._download_webpage(
-                base_url + 'get_video_info', video_id,
-                'Refetching age-gated info webpage',
-                'unable to download video info webpage', query={
-                    'video_id': video_id,
-                    'eurl': 'https://youtube.googleapis.com/v/' + video_id,
-                    'html5': 1,
-                    # See https://github.com/ytdl-org/youtube-dl/issues/29333#issuecomment-864049544
-                    'c': 'TVHTML5',
-                    'cver': '6.20180913',
-                }, fatal=False)
-            if video_info:
-                pr = self._parse_json(
-                    try_get(
-                        compat_parse_qs(video_info),
-                        lambda x: x['player_response'][0], compat_str) or '{}',
-                    video_id, fatal=False)
-                if pr and isinstance(pr, dict):
-                    player_response = pr
+        def is_agegated(playability):
+            if not isinstance(playability, dict):
+                return
+
+            if playability.get('desktopLegacyAgeGateReason'):
+                return True
+
+            reasons = filter(None, (playability.get(r) for r in ('status', 'reason')))
+            AGE_GATE_REASONS = (
+                'confirm your age', 'age-restricted', 'inappropriate',  # reason
+                'age_verification_required', 'age_check_required',  # status
+            )
+            return any(expected in reason for expected in AGE_GATE_REASONS for reason in reasons)
+
+        def get_playability_status(response):
+            return try_get(response, lambda x: x['playabilityStatus'], dict) or {}
+
+        playability_status = get_playability_status(player_response)
+        if (is_agegated(playability_status)
+                and int_or_none(self._downloader.params.get('age_limit'), default=18) >= 18):
+
+            self.report_age_confirmation()
+
+            # Thanks: https://github.com/yt-dlp/yt-dlp/pull/3233
+            pb_context = {'html5Preference': 'HTML5_PREF_WANTS'}
+            query = {
+                'playbackContext': {'contentPlaybackContext': {'html5Preference': 'HTML5_PREF_WANTS'}},
+                'contentCheckOk': True,
+                'racyCheckOk': True,
+                'context': {
+                    'client': {'clientName': 'TVHTML5_SIMPLY_EMBEDDED_PLAYER', 'clientVersion': '2.0', 'hl': 'en', 'clientScreen': 'EMBED'},
+                    'thirdParty': {'embedUrl': 'https://google.com'},
+                },
+                'videoId': video_id,
+            }
+            headers = {
+                'X-YouTube-Client-Name': '85',
+                'X-YouTube-Client-Version': '2.0',
+                'Origin': 'https://www.youtube.com'
+            }
+
+            video_info = self._call_api('player', query, video_id, fatal=False, headers=headers)
+            age_gate_status = get_playability_status(video_info)
+            if age_gate_status.get('status') == 'OK':
+                player_response = video_info
+                playability_status = age_gate_status
 
         trailer_video_id = try_get(
             playability_status,
@@ -1932,12 +2048,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             for thumbnail in (try_get(
                     container,
                     lambda x: x['thumbnail']['thumbnails'], list) or []):
-                thumbnail_url = thumbnail.get('url')
+                thumbnail_url = url_or_none(thumbnail.get('url'))
                 if not thumbnail_url:
                     continue
                 thumbnails.append({
                     'height': int_or_none(thumbnail.get('height')),
-                    'url': thumbnail_url,
+                    'url': update_url(thumbnail_url, query=None, fragment=None),
                     'width': int_or_none(thumbnail.get('width')),
                 })
             if thumbnails:
@@ -2142,6 +2258,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     sbr_tooltip = try_get(
                         vpir, lambda x: x['sentimentBar']['sentimentBarRenderer']['tooltip'])
                     if sbr_tooltip:
+                        # however dislike_count was hidden by YT, as if there could ever be dislikable content on YT
                         like_count, dislike_count = sbr_tooltip.split(' / ')
                         info.update({
                             'like_count': str_to_int(like_count),
@@ -2411,7 +2528,6 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'tags': list,
             'view_count': int,
             'like_count': int,
-            'dislike_count': int,
         },
         'params': {
             'skip_download': True,
@@ -2438,7 +2554,6 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'categories': ['News & Politics'],
             'tags': list,
             'like_count': int,
-            'dislike_count': int,
         },
         'params': {
             'skip_download': True,
@@ -2458,7 +2573,6 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'categories': ['News & Politics'],
             'tags': ['Cenk Uygur (TV Program Creator)', 'The Young Turks (Award-Winning Work)', 'Talk Show (TV Genre)'],
             'like_count': int,
-            'dislike_count': int,
         },
         'params': {
             'skip_download': True,
@@ -3043,8 +3157,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
 
     def _real_extract(self, url):
         item_id = self._match_id(url)
-        url = compat_urlparse.urlunparse(
-            compat_urlparse.urlparse(url)._replace(netloc='www.youtube.com'))
+        url = update_url(url, netloc='www.youtube.com')
         # Handle both video/playlist URLs
         qs = parse_qs(url)
         video_id = qs.get('v', [None])[0]
@@ -3178,7 +3291,6 @@ class YoutubeYtBeIE(InfoExtractor):
             'categories': ['Nonprofits & Activism'],
             'tags': list,
             'like_count': int,
-            'dislike_count': int,
         },
         'params': {
             'noplaylist': True,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e3c3ccff9..d5cc6386d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -4121,6 +4121,17 @@ def update_url_query(url, query):
         query=compat_urllib_parse_urlencode(qs, True)))
 
 
+def update_url(url, **kwargs):
+    """Replace URL components specified by kwargs
+       url: compat_str or parsed URL tuple
+       returns: compat_str"""
+    if not kwargs:
+        return compat_urlparse.urlunparse(url) if isinstance(url, tuple) else url
+    if not isinstance(url, tuple):
+        url = compat_urlparse.urlparse(url)
+    return compat_urlparse.urlunparse(url._replace(**kwargs))
+
+
 def update_Request(req, url=None, data=None, headers={}, query={}):
     req_headers = req.headers.copy()
     req_headers.update(headers)

From 30e986b83493f68bd4c2405b5f4d801891c9bdde Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 20 Jun 2022 23:15:20 +0100
Subject: [PATCH 591/860] [YouTube] Add `signatureTimestamp` for age-gate
 bypass

---
 youtube_dl/extractor/youtube.py | 34 +++++++++++++++++++++++++++++++--
 1 file changed, 32 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 65428528d..6c1cfe7f2 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1642,6 +1642,27 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             fmt['url'] = compat_urlparse.urlunparse(
                 parsed_fmt_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
 
+    # from yt-dlp, with tweaks
+    def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
+        """
+        Extract signatureTimestamp (sts)
+        Required to tell API what sig/player version is in use.
+        """
+        sts = int_or_none(ytcfg.get('STS')) if isinstance(ytcfg, dict) else None
+        if not sts:
+            # Attempt to extract from player
+            if player_url is None:
+                error_msg = 'Cannot extract signature timestamp without player_url.'
+                if fatal:
+                    raise ExtractorError(error_msg)
+                self._downloader.report_warning(error_msg)
+                return
+            code = self._get_player_code(video_id, player_url)
+            sts = int_or_none(self._search_regex(
+                r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code or '',
+                'JS player signature timestamp', group='sts', fatal=fatal))
+        return sts
+
     def _mark_watched(self, video_id, player_response):
         playback_url = url_or_none(try_get(
             player_response,
@@ -1766,6 +1787,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             webpage_url + '&bpctr=9999999999&has_verified=1', video_id, fatal=False)
 
         player_response = None
+        player_url = None
         if webpage:
             player_response = self._extract_yt_initial_variable(
                 webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE,
@@ -1799,8 +1821,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
             # Thanks: https://github.com/yt-dlp/yt-dlp/pull/3233
             pb_context = {'html5Preference': 'HTML5_PREF_WANTS'}
+
+            # Use signatureTimestamp if available
+            # Thanks https://github.com/ytdl-org/youtube-dl/issues/31034#issuecomment-1160718026
+            player_url = self._extract_player_url(webpage)
+            ytcfg = self._extract_ytcfg(video_id, webpage)
+            sts = self._extract_signature_timestamp(video_id, player_url, ytcfg)
+            if sts:
+                pb_context['signatureTimestamp'] = sts
+
             query = {
-                'playbackContext': {'contentPlaybackContext': {'html5Preference': 'HTML5_PREF_WANTS'}},
+                'playbackContext': {'contentPlaybackContext': pb_context},
                 'contentCheckOk': True,
                 'racyCheckOk': True,
                 'context': {
@@ -1901,7 +1932,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         formats = []
         itags = []
         itag_qualities = {}
-        player_url = None
         q = qualities(['tiny', 'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'])
         streaming_data = player_response.get('streamingData') or {}
         streaming_formats = streaming_data.get('formats') or []

From d6b14ba3163b255d0dd8d3b9ddf25d977b8262e7 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 4 Feb 2023 23:18:24 +0000
Subject: [PATCH 592/860] [test] Fix TestAgeRestriction

* age restriction may cause DownloadError
* update obsolete test URLs
[skip ci]
---
 test/test_age_restriction.py | 12 ++++++++----
 1 file changed, 8 insertions(+), 4 deletions(-)

diff --git a/test/test_age_restriction.py b/test/test_age_restriction.py
index 6f5513faa..db98494ab 100644
--- a/test/test_age_restriction.py
+++ b/test/test_age_restriction.py
@@ -11,6 +11,7 @@ from test.helper import try_rm
 
 
 from youtube_dl import YoutubeDL
+from youtube_dl.utils import DownloadError
 
 
 def _download_restricted(url, filename, age):
@@ -26,7 +27,10 @@ def _download_restricted(url, filename, age):
     ydl.add_default_info_extractors()
     json_filename = os.path.splitext(filename)[0] + '.info.json'
     try_rm(json_filename)
-    ydl.download([url])
+    try:
+        ydl.download([url])
+    except DownloadError:
+        try_rm(json_filename)
     res = os.path.exists(json_filename)
     try_rm(json_filename)
     return res
@@ -38,12 +42,12 @@ class TestAgeRestriction(unittest.TestCase):
         self.assertFalse(_download_restricted(url, filename, age))
 
     def test_youtube(self):
-        self._assert_restricted('07FYdnEawAQ', '07FYdnEawAQ.mp4', 10)
+        self._assert_restricted('HtVdAasjOgU', 'HtVdAasjOgU.mp4', 10)
 
     def test_youporn(self):
         self._assert_restricted(
-            'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
-            '505835.mp4', 2, old_age=25)
+            'https://www.youporn.com/watch/16715086/sex-ed-in-detention-18-asmr/',
+            '16715086.mp4', 2, old_age=25)
 
 
 if __name__ == '__main__':

From 249f2b631629471af5cfee2993e62de58c8f5990 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 5 Feb 2023 15:43:43 +0000
Subject: [PATCH 593/860] [compat] Systematise compat_ naming

[skip ci]
---
 test/test_compat.py  |   3 +-
 youtube_dl/compat.py | 221 +++++++++++++++++++++++++++----------------
 2 files changed, 139 insertions(+), 85 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 0986cff37..4dddd9a38 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -48,7 +48,8 @@ class TestCompat(unittest.TestCase):
 
     def test_all_present(self):
         import youtube_dl.compat
-        all_names = youtube_dl.compat.__all__
+        all_names = sorted(
+            youtube_dl.compat.__all__ + youtube_dl.compat.legacy)
         present_names = set(filter(
             lambda c: '_' in c and not c.startswith('_'),
             dir(youtube_dl.compat))) - set(['unicode_literals'])
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 28942a8c1..39551f810 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -21,6 +21,10 @@ import subprocess
 import sys
 import xml.etree.ElementTree
 
+# naming convention
+# 'compat_' + Python3_name.replace('.', '_')
+# other aliases exist for convenience and/or legacy
+
 # deal with critical unicode/str things first
 try:
     # Python 2
@@ -28,6 +32,7 @@ try:
         unicode, basestring, unichr
     )
     from .casefold import casefold as compat_casefold
+
 except NameError:
     compat_str, compat_basestring, compat_chr = (
         str, str, chr
@@ -53,16 +58,15 @@ try:
     import urllib.parse as compat_urllib_parse
 except ImportError:  # Python 2
     import urllib as compat_urllib_parse
+    import urlparse as _urlparse
+    for a in dir(_urlparse):
+        if not hasattr(compat_urllib_parse, a):
+            setattr(compat_urllib_parse, a, getattr(_urlparse, a))
+    del _urlparse
 
-try:
-    from urllib.parse import urlparse as compat_urllib_parse_urlparse
-except ImportError:  # Python 2
-    from urlparse import urlparse as compat_urllib_parse_urlparse
-
-try:
-    import urllib.parse as compat_urlparse
-except ImportError:  # Python 2
-    import urlparse as compat_urlparse
+# unfavoured aliases
+compat_urlparse = compat_urllib_parse
+compat_urllib_parse_urlparse = compat_urllib_parse.urlparse
 
 try:
     import urllib.response as compat_urllib_response
@@ -73,6 +77,7 @@ try:
     import http.cookiejar as compat_cookiejar
 except ImportError:  # Python 2
     import cookielib as compat_cookiejar
+compat_http_cookiejar = compat_cookiejar
 
 if sys.version_info[0] == 2:
     class compat_cookiejar_Cookie(compat_cookiejar.Cookie):
@@ -84,11 +89,13 @@ if sys.version_info[0] == 2:
             compat_cookiejar.Cookie.__init__(self, version, name, value, *args, **kwargs)
 else:
     compat_cookiejar_Cookie = compat_cookiejar.Cookie
+compat_http_cookiejar_Cookie = compat_cookiejar_Cookie
 
 try:
     import http.cookies as compat_cookies
 except ImportError:  # Python 2
     import Cookie as compat_cookies
+compat_http_cookies = compat_cookies
 
 if sys.version_info[0] == 2:
     class compat_cookies_SimpleCookie(compat_cookies.SimpleCookie):
@@ -98,6 +105,7 @@ if sys.version_info[0] == 2:
             return super(compat_cookies_SimpleCookie, self).load(rawdata)
 else:
     compat_cookies_SimpleCookie = compat_cookies.SimpleCookie
+compat_http_cookies_SimpleCookie = compat_cookies_SimpleCookie
 
 try:
     import html.entities as compat_html_entities
@@ -2351,16 +2359,19 @@ try:
     from urllib.error import HTTPError as compat_HTTPError
 except ImportError:  # Python 2
     from urllib2 import HTTPError as compat_HTTPError
+compat_urllib_HTTPError = compat_HTTPError
 
 try:
     from urllib.request import urlretrieve as compat_urlretrieve
 except ImportError:  # Python 2
     from urllib import urlretrieve as compat_urlretrieve
+compat_urllib_request_urlretrieve = compat_urlretrieve
 
 try:
     from html.parser import HTMLParser as compat_HTMLParser
 except ImportError:  # Python 2
     from HTMLParser import HTMLParser as compat_HTMLParser
+compat_html_parser_HTMLParser = compat_HTMLParser
 
 try:  # Python 2
     from HTMLParser import HTMLParseError as compat_HTMLParseError
@@ -2374,6 +2385,7 @@ except ImportError:  # Python <3.4
         # and uniform cross-version exception handling
         class compat_HTMLParseError(Exception):
             pass
+compat_html_parser_HTMLParseError = compat_HTMLParseError
 
 try:
     from subprocess import DEVNULL
@@ -2390,6 +2402,8 @@ try:
     from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
     from urllib.parse import unquote as compat_urllib_parse_unquote
     from urllib.parse import unquote_plus as compat_urllib_parse_unquote_plus
+    from urllib.parse import urlencode as compat_urllib_parse_urlencode
+    from urllib.parse import parse_qs as compat_parse_qs
 except ImportError:  # Python 2
     _asciire = (compat_urllib_parse._asciire if hasattr(compat_urllib_parse, '_asciire')
                 else re.compile(r'([\x00-\x7f]+)'))
@@ -2456,9 +2470,6 @@ except ImportError:  # Python 2
         string = string.replace('+', ' ')
         return compat_urllib_parse_unquote(string, encoding, errors)
 
-try:
-    from urllib.parse import urlencode as compat_urllib_parse_urlencode
-except ImportError:  # Python 2
     # Python 2 will choke in urlencode on mixture of byte and unicode strings.
     # Possible solutions are to either port it from python 3 with all
     # the friends or manually ensure input query contains only byte strings.
@@ -2480,7 +2491,62 @@ except ImportError:  # Python 2
         def encode_list(l):
             return [encode_elem(e) for e in l]
 
-        return compat_urllib_parse.urlencode(encode_elem(query), doseq=doseq)
+        return compat_urllib_parse._urlencode(encode_elem(query), doseq=doseq)
+
+    # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
+    # Python 2's version is apparently totally broken
+    def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
+                   encoding='utf-8', errors='replace'):
+        qs, _coerce_result = qs, compat_str
+        pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
+        r = []
+        for name_value in pairs:
+            if not name_value and not strict_parsing:
+                continue
+            nv = name_value.split('=', 1)
+            if len(nv) != 2:
+                if strict_parsing:
+                    raise ValueError('bad query field: %r' % (name_value,))
+                # Handle case of a control-name with no equal sign
+                if keep_blank_values:
+                    nv.append('')
+                else:
+                    continue
+            if len(nv[1]) or keep_blank_values:
+                name = nv[0].replace('+', ' ')
+                name = compat_urllib_parse_unquote(
+                    name, encoding=encoding, errors=errors)
+                name = _coerce_result(name)
+                value = nv[1].replace('+', ' ')
+                value = compat_urllib_parse_unquote(
+                    value, encoding=encoding, errors=errors)
+                value = _coerce_result(value)
+                r.append((name, value))
+        return r
+
+    def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
+                        encoding='utf-8', errors='replace'):
+        parsed_result = {}
+        pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
+                           encoding=encoding, errors=errors)
+        for name, value in pairs:
+            if name in parsed_result:
+                parsed_result[name].append(value)
+            else:
+                parsed_result[name] = [value]
+        return parsed_result
+
+    setattr(compat_urllib_parse, '_urlencode',
+            getattr(compat_urllib_parse, 'urlencode'))
+    for name, fix in (
+            ('unquote_to_bytes', compat_urllib_parse_unquote_to_bytes),
+            ('parse_unquote', compat_urllib_parse_unquote),
+            ('unquote_plus', compat_urllib_parse_unquote_plus),
+            ('urlencode', compat_urllib_parse_urlencode),
+            ('parse_qs', compat_parse_qs)):
+        setattr(compat_urllib_parse, name, fix)
+
+compat_urllib_parse_parse_qs = compat_parse_qs
 
 try:
     from urllib.request import DataHandler as compat_urllib_request_DataHandler
@@ -2520,6 +2586,7 @@ try:
     from xml.etree.ElementTree import ParseError as compat_xml_parse_error
 except ImportError:  # Python 2.6
     from xml.parsers.expat import ExpatError as compat_xml_parse_error
+compat_xml_etree_ElementTree_ParseError = compat_xml_parse_error
 
 etree = xml.etree.ElementTree
 
@@ -2533,10 +2600,11 @@ try:
     # xml.etree.ElementTree.Element is a method in Python <=2.6 and
     # the following will crash with:
     #  TypeError: isinstance() arg 2 must be a class, type, or tuple of classes and types
-    isinstance(None, xml.etree.ElementTree.Element)
+    isinstance(None, etree.Element)
     from xml.etree.ElementTree import Element as compat_etree_Element
 except TypeError:  # Python <=2.6
     from xml.etree.ElementTree import _ElementInterface as compat_etree_Element
+compat_xml_etree_ElementTree_Element = compat_etree_Element
 
 if sys.version_info[0] >= 3:
     def compat_etree_fromstring(text):
@@ -2592,6 +2660,7 @@ else:
             if k == uri or v == prefix:
                 del etree._namespace_map[k]
         etree._namespace_map[uri] = prefix
+compat_xml_etree_register_namespace = compat_etree_register_namespace
 
 if sys.version_info < (2, 7):
     # Here comes the crazy part: In 2.6, if the xpath is a unicode,
@@ -2603,53 +2672,6 @@ if sys.version_info < (2, 7):
 else:
     compat_xpath = lambda xpath: xpath
 
-try:
-    from urllib.parse import parse_qs as compat_parse_qs
-except ImportError:  # Python 2
-    # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
-    # Python 2's version is apparently totally broken
-
-    def _parse_qsl(qs, keep_blank_values=False, strict_parsing=False,
-                   encoding='utf-8', errors='replace'):
-        qs, _coerce_result = qs, compat_str
-        pairs = [s2 for s1 in qs.split('&') for s2 in s1.split(';')]
-        r = []
-        for name_value in pairs:
-            if not name_value and not strict_parsing:
-                continue
-            nv = name_value.split('=', 1)
-            if len(nv) != 2:
-                if strict_parsing:
-                    raise ValueError('bad query field: %r' % (name_value,))
-                # Handle case of a control-name with no equal sign
-                if keep_blank_values:
-                    nv.append('')
-                else:
-                    continue
-            if len(nv[1]) or keep_blank_values:
-                name = nv[0].replace('+', ' ')
-                name = compat_urllib_parse_unquote(
-                    name, encoding=encoding, errors=errors)
-                name = _coerce_result(name)
-                value = nv[1].replace('+', ' ')
-                value = compat_urllib_parse_unquote(
-                    value, encoding=encoding, errors=errors)
-                value = _coerce_result(value)
-                r.append((name, value))
-        return r
-
-    def compat_parse_qs(qs, keep_blank_values=False, strict_parsing=False,
-                        encoding='utf-8', errors='replace'):
-        parsed_result = {}
-        pairs = _parse_qsl(qs, keep_blank_values, strict_parsing,
-                           encoding=encoding, errors=errors)
-        for name, value in pairs:
-            if name in parsed_result:
-                parsed_result[name].append(value)
-            else:
-                parsed_result[name] = [value]
-        return parsed_result
-
 
 compat_os_name = os._name if os.name == 'java' else os.name
 
@@ -2774,6 +2796,8 @@ else:
     else:
         compat_expanduser = os.path.expanduser
 
+compat_os_path_expanduser = compat_expanduser
+
 
 if compat_os_name == 'nt' and sys.version_info < (3, 8):
     # os.path.realpath on Windows does not follow symbolic links
@@ -2785,6 +2809,8 @@ if compat_os_name == 'nt' and sys.version_info < (3, 8):
 else:
     compat_realpath = os.path.realpath
 
+compat_os_path_realpath = compat_realpath
+
 
 if sys.version_info < (3, 0):
     def compat_print(s):
@@ -2805,11 +2831,15 @@ if sys.version_info < (3, 0) and sys.platform == 'win32':
 else:
     compat_getpass = getpass.getpass
 
+compat_getpass_getpass = compat_getpass
+
+
 try:
     compat_input = raw_input
 except NameError:  # Python 3
     compat_input = input
 
+
 # Python < 2.6.5 require kwargs to be bytes
 try:
     def _testfunc(x):
@@ -2915,15 +2945,16 @@ else:
                 lines = _lines
         return _terminal_size(columns, lines)
 
+
 try:
     itertools.count(start=0, step=1)
     compat_itertools_count = itertools.count
 except TypeError:  # Python 2.6
     def compat_itertools_count(start=0, step=1):
-        n = start
         while True:
-            yield n
-            n += step
+            yield start
+            start += step
+
 
 if sys.version_info >= (3, 0):
     from tokenize import tokenize as compat_tokenize_tokenize
@@ -3075,6 +3106,8 @@ if sys.version_info < (3, 3):
 else:
     compat_b64decode = base64.b64decode
 
+compat_base64_b64decode = compat_b64decode
+
 
 if platform.python_implementation() == 'PyPy' and sys.pypy_version_info < (5, 4, 0):
     # PyPy2 prior to version 5.4.0 expects byte strings as Windows function
@@ -3094,30 +3127,53 @@ else:
         return ctypes.WINFUNCTYPE(*args, **kwargs)
 
 
-__all__ = [
+legacy = [
     'compat_HTMLParseError',
     'compat_HTMLParser',
     'compat_HTTPError',
-    'compat_Struct',
     'compat_b64decode',
+    'compat_cookiejar',
+    'compat_cookiejar_Cookie',
+    'compat_cookies',
+    'compat_cookies_SimpleCookie',
+    'compat_etree_Element',
+    'compat_etree_register_namespace',
+    'compat_expanduser',
+    'compat_getpass',
+    'compat_parse_qs',
+    'compat_realpath',
+    'compat_urllib_parse_parse_qs',
+    'compat_urllib_parse_unquote',
+    'compat_urllib_parse_unquote_plus',
+    'compat_urllib_parse_unquote_to_bytes',
+    'compat_urllib_parse_urlencode',
+    'compat_urllib_parse_urlparse',
+    'compat_urlparse',
+    'compat_urlretrieve',
+    'compat_xml_parse_error',
+]
+
+
+__all__ = [
+    'compat_html_parser_HTMLParseError',
+    'compat_html_parser_HTMLParser',
+    'compat_Struct',
+    'compat_base64_b64decode',
     'compat_basestring',
     'compat_casefold',
     'compat_chr',
     'compat_collections_abc',
     'compat_collections_chain_map',
-    'compat_cookiejar',
-    'compat_cookiejar_Cookie',
-    'compat_cookies',
-    'compat_cookies_SimpleCookie',
+    'compat_http_cookiejar',
+    'compat_http_cookiejar_Cookie',
+    'compat_http_cookies',
+    'compat_http_cookies_SimpleCookie',
     'compat_ctypes_WINFUNCTYPE',
-    'compat_etree_Element',
     'compat_etree_fromstring',
-    'compat_etree_register_namespace',
-    'compat_expanduser',
     'compat_filter',
     'compat_get_terminal_size',
     'compat_getenv',
-    'compat_getpass',
+    'compat_getpass_getpass',
     'compat_html_entities',
     'compat_html_entities_html5',
     'compat_http_client',
@@ -3131,11 +3187,11 @@ __all__ = [
     'compat_numeric_types',
     'compat_ord',
     'compat_os_name',
-    'compat_parse_qs',
+    'compat_os_path_expanduser',
+    'compat_os_path_realpath',
     'compat_print',
     'compat_re_Match',
     'compat_re_Pattern',
-    'compat_realpath',
     'compat_setenv',
     'compat_shlex_quote',
     'compat_shlex_split',
@@ -3147,17 +3203,14 @@ __all__ = [
     'compat_tokenize_tokenize',
     'compat_urllib_error',
     'compat_urllib_parse',
-    'compat_urllib_parse_unquote',
-    'compat_urllib_parse_unquote_plus',
-    'compat_urllib_parse_unquote_to_bytes',
-    'compat_urllib_parse_urlencode',
-    'compat_urllib_parse_urlparse',
     'compat_urllib_request',
     'compat_urllib_request_DataHandler',
     'compat_urllib_response',
-    'compat_urlparse',
-    'compat_urlretrieve',
-    'compat_xml_parse_error',
+    'compat_urllib_request_urlretrieve',
+    'compat_urllib_HTTPError',
+    'compat_xml_etree_ElementTree_Element',
+    'compat_xml_etree_ElementTree_ParseError',
+    'compat_xml_etree_register_namespace',
     'compat_xpath',
     'compat_zip',
     'workaround_optparse_bug9161',

From 90c9f789d94fc2c0b4c28c57ba2e0b2f09ef95e3 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 5 Feb 2023 13:46:43 +0000
Subject: [PATCH 594/860] [utils] Add parse_qs, update_url

[skip ci]
---
 youtube_dl/utils.py | 64 ++++++++++++++++++++++-----------------------
 1 file changed, 32 insertions(+), 32 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d5cc6386d..4edbfa27b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -42,6 +42,7 @@ from .compat import (
     compat_HTMLParser,
     compat_HTTPError,
     compat_basestring,
+    compat_casefold,
     compat_chr,
     compat_collections_abc,
     compat_cookiejar,
@@ -54,18 +55,18 @@ from .compat import (
     compat_integer_types,
     compat_kwargs,
     compat_os_name,
-    compat_parse_qs,
+    compat_re_Match,
     compat_shlex_quote,
     compat_str,
     compat_struct_pack,
     compat_struct_unpack,
     compat_urllib_error,
     compat_urllib_parse,
+    compat_urllib_parse_parse_qs as compat_parse_qs,
     compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse,
     compat_urllib_parse_unquote_plus,
     compat_urllib_request,
-    compat_urlparse,
     compat_xpath,
 )
 
@@ -80,12 +81,12 @@ def register_socks_protocols():
     # In Python < 2.6.5, urlsplit() suffers from bug https://bugs.python.org/issue7904
     # URLs with protocols not in urlparse.uses_netloc are not handled correctly
     for scheme in ('socks', 'socks4', 'socks4a', 'socks5'):
-        if scheme not in compat_urlparse.uses_netloc:
-            compat_urlparse.uses_netloc.append(scheme)
+        if scheme not in compat_urllib_parse.uses_netloc:
+            compat_urllib_parse.uses_netloc.append(scheme)
 
 
-# This is not clearly defined otherwise
-compiled_regex_type = type(re.compile(''))
+# Unfavoured alias
+compiled_regex_type = compat_re_Match
 
 
 def random_user_agent():
@@ -2725,7 +2726,7 @@ def make_socks_conn_class(base_class, socks_proxy):
     assert issubclass(base_class, (
         compat_http_client.HTTPConnection, compat_http_client.HTTPSConnection))
 
-    url_components = compat_urlparse.urlparse(socks_proxy)
+    url_components = compat_urllib_parse.urlparse(socks_proxy)
     if url_components.scheme.lower() == 'socks5':
         socks_type = ProxyType.SOCKS5
     elif url_components.scheme.lower() in ('socks', 'socks4'):
@@ -3673,7 +3674,7 @@ def remove_quotes(s):
 
 
 def url_basename(url):
-    path = compat_urlparse.urlparse(url).path
+    path = compat_urllib_parse.urlparse(url).path
     return path.strip('/').split('/')[-1]
 
 
@@ -3693,7 +3694,7 @@ def urljoin(base, path):
     if not isinstance(base, compat_str) or not re.match(
             r'^(?:https?:)?//', base):
         return None
-    return compat_urlparse.urljoin(base, path)
+    return compat_urllib_parse.urljoin(base, path)
 
 
 class HEADRequest(compat_urllib_request.Request):
@@ -4091,6 +4092,10 @@ def escape_url(url):
     ).geturl()
 
 
+def parse_qs(url):
+    return compat_parse_qs(compat_urllib_parse.urlparse(url).query)
+
+
 def read_batch_urls(batch_fd):
     def fixup(url):
         if not isinstance(url, compat_str):
@@ -4111,25 +4116,28 @@ def urlencode_postdata(*args, **kargs):
     return compat_urllib_parse_urlencode(*args, **kargs).encode('ascii')
 
 
-def update_url_query(url, query):
-    if not query:
-        return url
-    parsed_url = compat_urlparse.urlparse(url)
-    qs = compat_parse_qs(parsed_url.query)
-    qs.update(query)
-    return compat_urlparse.urlunparse(parsed_url._replace(
-        query=compat_urllib_parse_urlencode(qs, True)))
-
-
 def update_url(url, **kwargs):
     """Replace URL components specified by kwargs
        url: compat_str or parsed URL tuple
-       returns: compat_str"""
+       if query_update is in kwargs, update query with
+       its value instead of replacing (overrides any `query`)
+       returns: compat_str
+    """
     if not kwargs:
-        return compat_urlparse.urlunparse(url) if isinstance(url, tuple) else url
+        return compat_urllib_parse.urlunparse(url) if isinstance(url, tuple) else url
     if not isinstance(url, tuple):
-        url = compat_urlparse.urlparse(url)
-    return compat_urlparse.urlunparse(url._replace(**kwargs))
+        url = compat_urllib_parse.urlparse(url)
+    query = kwargs.pop('query_update', None)
+    if query:
+        qs = compat_parse_qs(url.query)
+        qs.update(query)
+        kwargs['query'] = compat_urllib_parse_urlencode(qs, True)
+        kwargs = compat_kwargs(kwargs)
+    return compat_urllib_parse.urlunparse(url._replace(**kwargs))
+
+
+def update_url_query(url, query):
+    return update_url(url, query_update=query)
 
 
 def update_Request(req, url=None, data=None, headers={}, query={}):
@@ -5597,7 +5605,7 @@ class PerRequestProxyHandler(compat_urllib_request.ProxyHandler):
 
         if proxy == '__noproxy__':
             return None  # No Proxy
-        if compat_urlparse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
+        if compat_urllib_parse.urlparse(proxy).scheme.lower() in ('socks', 'socks4', 'socks4a', 'socks5'):
             req.add_header('Ytdl-socks-proxy', proxy)
             # youtube-dl's http/https handlers do wrapping the socket with socks
             return None
@@ -6035,14 +6043,6 @@ def traverse_obj(obj, *paths, **kwargs):
     str = compat_str
 
     is_sequence = lambda x: isinstance(x, compat_collections_abc.Sequence) and not isinstance(x, (str, bytes))
-    # stand-in until compat_re_Match is added
-    compat_re_Match = type(re.match('a', 'a'))
-    # stand-in until casefold.py is added
-    try:
-        ''.casefold()
-        compat_casefold = lambda s: s.casefold()
-    except AttributeError:
-        compat_casefold = lambda s: s.lower()
     casefold = lambda k: compat_casefold(k) if isinstance(k, str) else k
 
     if isinstance(expected_type, type):

From 4e04f104994c5dac2cb74b64ba7725716ce939d7 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 6 Feb 2023 15:50:28 +0000
Subject: [PATCH 595/860] [compat] Update test_compat

[skip ci]
---
 test/test_compat.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index 4dddd9a38..e233b1ae1 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -50,9 +50,9 @@ class TestCompat(unittest.TestCase):
         import youtube_dl.compat
         all_names = sorted(
             youtube_dl.compat.__all__ + youtube_dl.compat.legacy)
-        present_names = set(filter(
+        present_names = set(map(compat_str, filter(
             lambda c: '_' in c and not c.startswith('_'),
-            dir(youtube_dl.compat))) - set(['unicode_literals'])
+            dir(youtube_dl.compat)))) - set(['unicode_literals'])
         self.assertEqual(all_names, sorted(present_names))
 
     def test_compat_urllib_parse_unquote(self):

From bafb6dec72865cc494feb35ecc94481c30a81069 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 6 Feb 2023 16:19:21 +0000
Subject: [PATCH 596/860] [YouTube] Refresh compat/utils usage * import
 parse_qs() * import parse_qs in lazy_extractors (clears old TODO) * clean up
 old compiled lazy_extractors for Py2 * use update_url()

---
 devscripts/make_lazy_extractors.py | 10 ++++-
 test/test_execution.py             | 12 +++---
 youtube_dl/extractor/youtube.py    | 61 +++++++++++-------------------
 3 files changed, 39 insertions(+), 44 deletions(-)

diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py
index 878ae72b1..edc19183d 100644
--- a/devscripts/make_lazy_extractors.py
+++ b/devscripts/make_lazy_extractors.py
@@ -13,6 +13,11 @@ sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
 lazy_extractors_filename = sys.argv[1]
 if os.path.exists(lazy_extractors_filename):
     os.remove(lazy_extractors_filename)
+# Py2: may be confused by leftover lazy_extractors.pyc
+try:
+    os.remove(lazy_extractors_filename + 'c')
+except OSError:
+    pass
 
 from youtube_dl.extractor import _ALL_CLASSES
 from youtube_dl.extractor.common import InfoExtractor, SearchInfoExtractor
@@ -22,7 +27,10 @@ with open('devscripts/lazy_load_template.py', 'rt') as f:
 
 module_contents = [
     module_template + '\n' + getsource(InfoExtractor.suitable) + '\n',
-    'class LazyLoadSearchExtractor(LazyLoadExtractor):\n    pass\n']
+    'class LazyLoadSearchExtractor(LazyLoadExtractor):\n    pass\n',
+    # needed for suitable() methods of Youtube extractor (see #28780)
+    'from youtube_dl.utils import parse_qs\n',
+]
 
 ie_template = '''
 class {name}({bases}):
diff --git a/test/test_execution.py b/test/test_execution.py
index 32948d93e..704e14612 100644
--- a/test/test_execution.py
+++ b/test/test_execution.py
@@ -40,14 +40,16 @@ class TestExecution(unittest.TestCase):
         self.assertFalse(stderr)
 
     def test_lazy_extractors(self):
+        lazy_extractors = 'youtube_dl/extractor/lazy_extractors.py'
         try:
-            subprocess.check_call([sys.executable, 'devscripts/make_lazy_extractors.py', 'youtube_dl/extractor/lazy_extractors.py'], cwd=rootDir, stdout=_DEV_NULL)
+            subprocess.check_call([sys.executable, 'devscripts/make_lazy_extractors.py', lazy_extractors], cwd=rootDir, stdout=_DEV_NULL)
             subprocess.check_call([sys.executable, 'test/test_all_urls.py'], cwd=rootDir, stdout=_DEV_NULL)
         finally:
-            try:
-                os.remove('youtube_dl/extractor/lazy_extractors.py')
-            except (IOError, OSError):
-                pass
+            for x in ['', 'c'] if sys.version_info[0] < 3 else ['']:
+                try:
+                    os.remove(lazy_extractors + x)
+                except (IOError, OSError):
+                    pass
 
 
 if __name__ == '__main__':
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6c1cfe7f2..6c70a98d1 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -14,12 +14,11 @@ from ..compat import (
     compat_chr,
     compat_HTTPError,
     compat_map as map,
-    compat_parse_qs,
     compat_str,
+    compat_urllib_parse,
+    compat_urllib_parse_parse_qs as compat_parse_qs,
     compat_urllib_parse_unquote_plus,
-    compat_urllib_parse_urlencode,
     compat_urllib_parse_urlparse,
-    compat_urlparse,
 )
 from ..jsinterp import JSInterpreter
 from ..utils import (
@@ -33,6 +32,7 @@ from ..utils import (
     mimetype2ext,
     parse_codecs,
     parse_duration,
+    parse_qs,
     qualities,
     remove_start,
     smuggle_url,
@@ -50,10 +50,6 @@ from ..utils import (
 )
 
 
-def parse_qs(url):
-    return compat_urlparse.parse_qs(compat_urlparse.urlparse(url).query)
-
-
 class YoutubeBaseInfoExtractor(InfoExtractor):
     """Provide base functions for Youtube extractors"""
     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
@@ -636,6 +632,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
                 'duration': 142,
                 'uploader': 'The Witcher',
+                'uploader_id': 'WitcherGame',
+                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
                 'upload_date': '20140605',
                 'thumbnail': 'https://i.ytimg.com/vi/HtVdAasjOgU/maxresdefault.jpg',
                 'age_limit': 18,
@@ -671,7 +669,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             },
         },
         {
-            'note': 'Age-gated video embedable only with clientScreen=EMBED',
+            'note': 'Age-gated video embeddable only with clientScreen=EMBED',
             'url': 'https://youtube.com/watch?v=Tq92D6wQ1mg',
             'info_dict': {
                 'id': 'Tq92D6wQ1mg',
@@ -1392,11 +1390,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     @classmethod
     def suitable(cls, url):
-        # Hack for lazy extractors until more generic solution is implemented
-        # (see #28780)
-        from .youtube import parse_qs
-        qs = parse_qs(url)
-        if qs.get('list', [None])[0]:
+        if parse_qs(url).get('list', [None])[0]:
             return False
         return super(YoutubeIE, cls).suitable(url)
 
@@ -1546,7 +1540,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         if player_url.startswith('//'):
             player_url = 'https:' + player_url
         elif not re.match(r'https?://', player_url):
-            player_url = compat_urlparse.urljoin(
+            player_url = compat_urllib_parse.urljoin(
                 'https://www.youtube.com', player_url)
         return player_url
 
@@ -1628,9 +1622,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     def _unthrottle_format_urls(self, video_id, player_url, formats):
         for fmt in formats:
-            parsed_fmt_url = compat_urlparse.urlparse(fmt['url'])
-            qs = compat_urlparse.parse_qs(parsed_fmt_url.query)
-            n_param = qs.get('n')
+            parsed_fmt_url = compat_urllib_parse.urlparse(fmt['url'])
+            n_param = compat_parse_qs(parsed_fmt_url.query).get('n')
             if not n_param:
                 continue
             n_param = n_param[-1]
@@ -1638,9 +1631,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if n_response is None:
                 # give up if descrambling failed
                 break
-            qs['n'] = [n_response]
-            fmt['url'] = compat_urlparse.urlunparse(
-                parsed_fmt_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
+            fmt['url'] = update_url(
+                parsed_fmt_url, query_update={'n': [n_response]})
 
     # from yt-dlp, with tweaks
     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
@@ -1669,20 +1661,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             lambda x: x['playbackTracking']['videostatsPlaybackUrl']['baseUrl']))
         if not playback_url:
             return
-        parsed_playback_url = compat_urlparse.urlparse(playback_url)
-        qs = compat_urlparse.parse_qs(parsed_playback_url.query)
 
         # cpn generation algorithm is reverse engineered from base.js.
         # In fact it works even with dummy cpn.
         CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
         cpn = ''.join((CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16)))
 
-        qs.update({
-            'ver': ['2'],
-            'cpn': [cpn],
-        })
-        playback_url = compat_urlparse.urlunparse(
-            parsed_playback_url._replace(query=compat_urllib_parse_urlencode(qs, True)))
+        playback_url = update_url(
+            playback_url, query_update={
+                'ver': ['2'],
+                'cpn': [cpn],
+            })
 
         self._download_webpage(
             playback_url, video_id, 'Marking watched',
@@ -2075,9 +2064,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         thumbnails = []
         for container in (video_details, microformat):
-            for thumbnail in (try_get(
+            for thumbnail in try_get(
                     container,
-                    lambda x: x['thumbnail']['thumbnails'], list) or []):
+                    lambda x: x['thumbnail']['thumbnails'], list) or []:
                 thumbnail_url = url_or_none(thumbnail.get('url'))
                 if not thumbnail_url:
                     continue
@@ -3287,11 +3276,7 @@ class YoutubePlaylistIE(InfoExtractor):
     def suitable(cls, url):
         if YoutubeTabIE.suitable(url):
             return False
-        # Hack for lazy extractors until more generic solution is implemented
-        # (see #28780)
-        from .youtube import parse_qs
-        qs = parse_qs(url)
-        if qs.get('v', [None])[0]:
+        if parse_qs(url).get('v', [None])[0]:
             return False
         return super(YoutubePlaylistIE, cls).suitable(url)
 
@@ -3430,9 +3415,9 @@ class YoutubeSearchURLIE(YoutubeBaseInfoExtractor):
     }]
 
     def _real_extract(self, url):
-        qs = compat_parse_qs(compat_urllib_parse_urlparse(url).query)
-        query = (qs.get('search_query') or qs.get('q'))[0]
-        params = qs.get('sp', ('',))[0]
+        qs = parse_qs(url)
+        query = (qs.get('search_query') or qs.get('q'))[-1]
+        params = qs.get('sp', ('',))[-1]
         return self.playlist_result(self._search_results(query, params), query, query)
 
 

From e8198c517b70301dd5a459927b5d5976304d6482 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 8 Feb 2023 18:16:51 +0000
Subject: [PATCH 597/860] [YouTube] Fix tests

---
 youtube_dl/extractor/youtube.py | 55 ++++++++++++++++++++++++++++-----
 1 file changed, 48 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6c70a98d1..ba0f5c8b6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -27,6 +27,8 @@ from ..utils import (
     dict_get,
     error_to_compat_str,
     float_or_none,
+    extract_attributes,
+    get_element_by_attribute,
     int_or_none,
     js_to_json,
     mimetype2ext,
@@ -38,6 +40,7 @@ from ..utils import (
     smuggle_url,
     str_or_none,
     str_to_int,
+    traverse_obj,
     try_get,
     unescapeHTML,
     unified_strdate,
@@ -656,6 +659,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
                 'duration': 177,
                 'uploader': 'FlyingKitty',
+                'uploader_id': 'FlyingKitty900',
                 'upload_date': '20200408',
                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
                 'age_limit': 18,
@@ -678,6 +682,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:17eccca93a786d51bc67646756894066',
                 'duration': 106,
                 'uploader': 'Projekt Melody',
+                'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
                 'upload_date': '20191227',
                 'age_limit': 18,
                 'thumbnail': 'https://i.ytimg.com/vi/Tq92D6wQ1mg/sddefault.jpg',
@@ -929,16 +934,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'id': 'lsguqyKfVQg',
                 'ext': 'mp4',
                 'title': '{dark walk}; Loki/AC/Dishonored; collab w/Elflover21',
-                'alt_title': 'Dark Walk - Position Music',
+                'alt_title': 'Dark Walk',
                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
                 'duration': 133,
                 'upload_date': '20151119',
                 'uploader_id': 'IronSoulElf',
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
                 'uploader': 'IronSoulElf',
-                'creator': 'Todd Haberman,  Daniel Law Heath and Aaron Kaplan',
-                'track': 'Dark Walk - Position Music',
-                'artist': 'Todd Haberman,  Daniel Law Heath and Aaron Kaplan',
+                'creator': r're:Todd Haberman[;,]\s+Daniel Law Heath and Aaron Kaplan',
+                'track': 'Dark Walk',
+                'artist': r're:Todd Haberman[;,]\s+Daniel Law Heath and Aaron Kaplan',
                 'album': 'Position Music - Production Music Vol. 143 - Dark Walk',
             },
             'params': {
@@ -2091,7 +2096,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             or microformat.get('lengthSeconds')) \
             or parse_duration(search_meta('duration'))
         is_live = video_details.get('isLive')
-        owner_profile_url = microformat.get('ownerProfileUrl')
+
+        def gen_owner_profile_url():
+            yield microformat.get('ownerProfileUrl')
+            yield extract_attributes(self._search_regex(
+                r'''(?s)(<link\b[^>]+\bitemprop\s*=\s*("|')url\2[^>]*>)''',
+                get_element_by_attribute('itemprop', 'author', webpage),
+                'owner_profile_url', default='')).get('href')
+
+        owner_profile_url = next(
+            (x for x in map(url_or_none, gen_owner_profile_url()) if x),
+            None)
 
         if not player_url:
             player_url = self._extract_player_url(webpage)
@@ -2176,6 +2191,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         info[d_k] = parse_duration(query[k][0])
 
         if video_description:
+            # Youtube Music Auto-generated description
             mobj = re.search(r'(?s)(?P<track>[^·\n]+)·(?P<artist>[^\n]+)\n+(?P<album>[^\n]+)(?:.+?℗\s*(?P<release_year>\d{4})(?!\d))?(?:.+?Released on\s*:\s*(?P<release_date>\d{4}-\d{2}-\d{2}))?(.+?\nArtist\s*:\s*(?P<clean_artist>[^\n]+))?.+\nAuto-generated by YouTube\.\s*$', video_description)
             if mobj:
                 release_year = mobj.group('release_year')
@@ -2250,7 +2266,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                                 lambda x: x['superTitleIcon']['iconType']) == 'LOCATION_PIN':
                             info['location'] = stl
                         else:
-                            mobj = re.search(r'(.+?)\s*S(\d+)\s*•\s*E(\d+)', stl)
+                            # •? doesn't match, but [•]? does; \xa0 = non-breaking space
+                            mobj = re.search(r'([^\xa0\s].*?)[\xa0\s]*S(\d+)[\xa0\s]*[•]?[\xa0\s]*E(\d+)', stl)
                             if mobj:
                                 info.update({
                                     'series': mobj.group(1),
@@ -2261,7 +2278,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             vpir,
                             lambda x: x['videoActions']['menuRenderer']['topLevelButtons'],
                             list) or []):
-                        tbr = tlb.get('toggleButtonRenderer') or {}
+                        tbr = traverse_obj(tlb, ('segmentedLikeDislikeButtonRenderer', 'likeButton', 'toggleButtonRenderer'), 'toggleButtonRenderer') or {}
                         for getter, regex in [(
                                 lambda x: x['defaultText']['accessibility']['accessibilityData'],
                                 r'(?P<count>[\d,]+)\s*(?P<type>(?:dis)?like)'), ([
@@ -2315,6 +2332,30 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             elif mrr_title == 'Song':
                                 info['track'] = mrr_contents_text
 
+            # this is not extraction but spelunking!
+            carousel_lockups = traverse_obj(
+                initial_data,
+                ('engagementPanels', Ellipsis, 'engagementPanelSectionListRenderer',
+                 'content', 'structuredDescriptionContentRenderer', 'items', Ellipsis,
+                 'videoDescriptionMusicSectionRenderer', 'carouselLockups', Ellipsis),
+                expected_type=dict) or []
+            # try to reproduce logic from metadataRowContainerRenderer above (if it still is)
+            fields = (('ALBUM', 'album'), ('ARTIST', 'artist'), ('SONG', 'track'), ('LICENSES', 'license'))
+            # multiple_songs ?
+            if len(carousel_lockups) > 1:
+                fields = fields[-1:]
+            for info_row in traverse_obj(
+                    carousel_lockups,
+                    (0, 'carouselLockupRenderer', 'infoRows', Ellipsis, 'infoRowRenderer'),
+                    expected_type=dict):
+                row_title = traverse_obj(info_row, ('title', 'simpleText'))
+                row_text = traverse_obj(info_row, 'defaultMetadata', 'expandedMetadata', expected_type=get_text)
+                if not row_text:
+                    continue
+                for name, field in fields:
+                    if name == row_title and not info.get(field):
+                        info[field] = row_text
+
         for s_k, d_k in [('artist', 'creator'), ('track', 'alt_title')]:
             v = info.get(s_k)
             if v:

From f33923cba7670ea2e82f233c1f88210eb41f7c3b Mon Sep 17 00:00:00 2001
From: Valentin Metz <31850924+Valentin-Metz@users.noreply.github.com>
Date: Thu, 9 Feb 2023 12:25:28 +0100
Subject: [PATCH 598/860] [rbgtum] Add new extractor (#31305)

* [rbgtum] Add new extractor

* Small update, force CI

---------

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/extractors.py |  4 ++
 youtube_dl/extractor/rbgtum.py     | 97 ++++++++++++++++++++++++++++++
 2 files changed, 101 insertions(+)
 create mode 100644 youtube_dl/extractor/rbgtum.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 96b27b179..dfaef0cc3 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1010,6 +1010,10 @@ from .raywenderlich import (
     RayWenderlichIE,
     RayWenderlichCourseIE,
 )
+from .rbgtum import (
+    RbgTumIE,
+    RbgTumCourseIE,
+)
 from .rbmaradio import RBMARadioIE
 from .rds import RDSIE
 from .redbulltv import (
diff --git a/youtube_dl/extractor/rbgtum.py b/youtube_dl/extractor/rbgtum.py
new file mode 100644
index 000000000..da48ebbc4
--- /dev/null
+++ b/youtube_dl/extractor/rbgtum.py
@@ -0,0 +1,97 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+
+
+class RbgTumIE(InfoExtractor):
+    _VALID_URL = r'https://live\.rbg\.tum\.de/w/(?P<id>.+)'
+    _TESTS = [{
+        # Combined view
+        'url': 'https://live.rbg.tum.de/w/cpp/22128',
+        'md5': '53a5e7b3e07128e33bbf36687fe1c08f',
+        'info_dict': {
+            'id': 'cpp/22128',
+            'ext': 'mp4',
+            'title': 'Lecture: October 18. 2022',
+            'series': 'Concepts of C++ programming (IN2377)',
+        }
+    }, {
+        # Presentation only
+        'url': 'https://live.rbg.tum.de/w/I2DL/12349/PRES',
+        'md5': '36c584272179f3e56b0db5d880639cba',
+        'info_dict': {
+            'id': 'I2DL/12349/PRES',
+            'ext': 'mp4',
+            'title': 'Lecture 3: Introduction to Neural Networks',
+            'series': 'Introduction to Deep Learning (IN2346)',
+        }
+    }, {
+        # Camera only
+        'url': 'https://live.rbg.tum.de/w/fvv-info/16130/CAM',
+        'md5': 'e04189d92ff2f56aedf5cede65d37aad',
+        'info_dict': {
+            'id': 'fvv-info/16130/CAM',
+            'ext': 'mp4',
+            'title': 'Fachschaftsvollversammlung',
+            'series': 'Fachschaftsvollversammlung Informatik',
+        }
+    }, ]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+
+        m3u8 = self._html_search_regex(r'(https://.+?\.m3u8)', webpage, 'm3u8')
+        lecture_title = self._html_search_regex(r'(?si)<h1.*?>(.*)</h1>', webpage, 'title')
+        lecture_series_title = self._html_search_regex(
+            r'(?s)<title\b[^>]*>\s*(?:TUM-Live\s\|\s?)?([^:]+):?.*?</title>', webpage, 'series')
+
+        formats = self._extract_m3u8_formats(m3u8, video_id, 'mp4', entry_protocol='m3u8_native', m3u8_id='hls')
+        self._sort_formats(formats)
+
+        return {
+            'id': video_id,
+            'title': lecture_title,
+            'series': lecture_series_title,
+            'formats': formats,
+        }
+
+
+class RbgTumCourseIE(InfoExtractor):
+    _VALID_URL = r'https://live\.rbg\.tum\.de/course/(?P<id>.+)'
+    _TESTS = [{
+        'url': 'https://live.rbg.tum.de/course/2022/S/fpv',
+        'info_dict': {
+            'title': 'Funktionale Programmierung und Verifikation (IN0003)',
+            'id': '2022/S/fpv',
+        },
+        'params': {
+            'noplaylist': False,
+        },
+        'playlist_count': 13,
+    }, {
+        'url': 'https://live.rbg.tum.de/course/2022/W/set',
+        'info_dict': {
+            'title': 'SET FSMPIC',
+            'id': '2022/W/set',
+        },
+        'params': {
+            'noplaylist': False,
+        },
+        'playlist_count': 6,
+    }, ]
+
+    def _real_extract(self, url):
+        course_id = self._match_id(url)
+        webpage = self._download_webpage(url, course_id)
+
+        lecture_series_title = self._html_search_regex(r'(?si)<h1.*?>(.*)</h1>', webpage, 'title')
+
+        lecture_urls = []
+        for lecture_url in re.findall(r'(?i)href="/w/(.+)(?<!/cam)(?<!/pres)(?<!/chat)"', webpage):
+            lecture_urls.append(self.url_result('https://live.rbg.tum.de/w/' + lecture_url, ie=RbgTumIE.ie_key()))
+
+        return self.playlist_result(lecture_urls, course_id, lecture_series_title)

From 33db85c571304bbd6863e3407ad8d08764c9e53b Mon Sep 17 00:00:00 2001
From: teddy171 <teddy171@qq.com>
Date: Fri, 10 Feb 2023 04:19:27 +0800
Subject: [PATCH 599/860] [feat]: Add support to external downloader aria2p
 (#31500)

* feat: add class Aria2pFD

* feat: create call_downloader function

* feat: a colorful download interface to aria2pFD

* feat: change value name

* Apply suggestions from code review

Co-authored-by: dirkf <fieldhouse@gmx.net>

* Typo in suggestion

* fix: remove unused value

* fix: add not function to return value(0 is normal); add total_seconds to download.eta(timedelta object); add waiting status when hook progress

* fix: remove unuse method ..utils.format_bytes

* fix: be up to flake8

* fix: be up to flake8

* Apply suggestions from code review

* [feat] test external downloader aria2p

* [feat] test external downloader aria2p

* [fix] test_external_downloader.py

* Apply suggestions from code review

Co-authored-by: dirkf <fieldhouse@gmx.net>

* Apply suggestions from code review

Co-authored-by: dirkf <fieldhouse@gmx.net>

* Update test/test_external_downloader.py

Co-authored-by: dirkf <fieldhouse@gmx.net>

* Update test/test_external_downloader.py

Co-authored-by: dirkf <fieldhouse@gmx.net>

* Update youtube_dl/downloader/external.py

Co-authored-by: dirkf <fieldhouse@gmx.net>

* refactoring code and fix bugs

* Apply suggestions from code review

* Rename test_external_downloader.py to test_downloader_external.py

---------

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 test/helper.py                    |  11 +++
 test/test_downloader_external.py  | 115 ++++++++++++++++++++++++++++++
 test/test_downloader_http.py      |  17 ++---
 test/test_http.py                 |  16 ++---
 youtube_dl/downloader/external.py |  58 +++++++++++++++
 5 files changed, 193 insertions(+), 24 deletions(-)
 create mode 100644 test/test_downloader_external.py

diff --git a/test/helper.py b/test/helper.py
index c6a2f0667..883b2e877 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -89,6 +89,17 @@ class FakeYDL(YoutubeDL):
         self.report_warning = types.MethodType(report_warning, self)
 
 
+class FakeLogger(object):
+    def debug(self, msg):
+        pass
+
+    def warning(self, msg):
+        pass
+
+    def error(self, msg):
+        pass
+
+
 def gettestcases(include_onlymatching=False):
     for ie in youtube_dl.extractor.gen_extractors():
         for tc in ie.get_testcases(include_onlymatching):
diff --git a/test/test_downloader_external.py b/test/test_downloader_external.py
new file mode 100644
index 000000000..c0239502b
--- /dev/null
+++ b/test/test_downloader_external.py
@@ -0,0 +1,115 @@
+#!/usr/bin/env python
+# coding: utf-8
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import re
+import sys
+import subprocess
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+from test.helper import (
+    FakeLogger,
+    http_server_port,
+    try_rm,
+)
+from youtube_dl import YoutubeDL
+from youtube_dl.compat import compat_http_server
+from youtube_dl.utils import encodeFilename
+from youtube_dl.downloader.external import Aria2pFD
+import threading
+
+TEST_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+TEST_SIZE = 10 * 1024
+
+
+class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
+    def log_message(self, format, *args):
+        pass
+
+    def send_content_range(self, total=None):
+        range_header = self.headers.get('Range')
+        start = end = None
+        if range_header:
+            mobj = re.match(r'bytes=(\d+)-(\d+)', range_header)
+            if mobj:
+                start, end = (int(mobj.group(i)) for i in (1, 2))
+        valid_range = start is not None and end is not None
+        if valid_range:
+            content_range = 'bytes %d-%d' % (start, end)
+            if total:
+                content_range += '/%d' % total
+            self.send_header('Content-Range', content_range)
+        return (end - start + 1) if valid_range else total
+
+    def serve(self, range=True, content_length=True):
+        self.send_response(200)
+        self.send_header('Content-Type', 'video/mp4')
+        size = TEST_SIZE
+        if range:
+            size = self.send_content_range(TEST_SIZE)
+        if content_length:
+            self.send_header('Content-Length', size)
+        self.end_headers()
+        self.wfile.write(b'#' * size)
+
+    def do_GET(self):
+        if self.path == '/regular':
+            self.serve()
+        elif self.path == '/no-content-length':
+            self.serve(content_length=False)
+        elif self.path == '/no-range':
+            self.serve(range=False)
+        elif self.path == '/no-range-no-content-length':
+            self.serve(range=False, content_length=False)
+        else:
+            assert False, 'unrecognised server path'
+
+
+@unittest.skipUnless(Aria2pFD.available(), 'aria2p module not found')
+class TestAria2pFD(unittest.TestCase):
+    def setUp(self):
+        self.httpd = compat_http_server.HTTPServer(
+            ('127.0.0.1', 0), HTTPTestRequestHandler)
+        self.port = http_server_port(self.httpd)
+        self.server_thread = threading.Thread(target=self.httpd.serve_forever)
+        self.server_thread.daemon = True
+        self.server_thread.start()
+
+    def download(self, params, ep):
+        with subprocess.Popen(
+            ['aria2c', '--enable-rpc'],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.DEVNULL
+        ) as process:
+            if not process.poll():
+                filename = 'testfile.mp4'
+                params['logger'] = FakeLogger()
+                params['outtmpl'] = filename
+                ydl = YoutubeDL(params)
+                try_rm(encodeFilename(filename))
+                self.assertEqual(ydl.download(['http://127.0.0.1:%d/%s' % (self.port, ep)]), 0)
+                self.assertEqual(os.path.getsize(encodeFilename(filename)), TEST_SIZE)
+                try_rm(encodeFilename(filename))
+            process.kill()
+
+    def download_all(self, params):
+        for ep in ('regular', 'no-content-length', 'no-range', 'no-range-no-content-length'):
+            self.download(params, ep)
+
+    def test_regular(self):
+        self.download_all({'external_downloader': 'aria2p'})
+
+    def test_chunked(self):
+        self.download_all({
+            'external_downloader': 'aria2p',
+            'http_chunk_size': 1000,
+        })
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/test_downloader_http.py b/test/test_downloader_http.py
index 750472281..4e6d7a2a0 100644
--- a/test/test_downloader_http.py
+++ b/test/test_downloader_http.py
@@ -9,7 +9,11 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import http_server_port, try_rm
+from test.helper import (
+    FakeLogger,
+    http_server_port,
+    try_rm,
+)
 from youtube_dl import YoutubeDL
 from youtube_dl.compat import compat_http_server
 from youtube_dl.downloader.http import HttpFD
@@ -66,17 +70,6 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
             assert False
 
 
-class FakeLogger(object):
-    def debug(self, msg):
-        pass
-
-    def warning(self, msg):
-        pass
-
-    def error(self, msg):
-        pass
-
-
 class TestHttpFD(unittest.TestCase):
     def setUp(self):
         self.httpd = compat_http_server.HTTPServer(
diff --git a/test/test_http.py b/test/test_http.py
index 3ee0a5dda..487a9bc77 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -8,7 +8,10 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import http_server_port
+from test.helper import (
+    FakeLogger,
+    http_server_port,
+)
 from youtube_dl import YoutubeDL
 from youtube_dl.compat import compat_http_server, compat_urllib_request
 import ssl
@@ -52,17 +55,6 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
             assert False
 
 
-class FakeLogger(object):
-    def debug(self, msg):
-        pass
-
-    def warning(self, msg):
-        pass
-
-    def error(self, msg):
-        pass
-
-
 class TestHTTP(unittest.TestCase):
     def setUp(self):
         self.httpd = compat_http_server.HTTPServer(
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index a06ab2e50..bffcd10b6 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -200,6 +200,64 @@ class Aria2cFD(ExternalFD):
         return cmd
 
 
+class Aria2pFD(ExternalFD):
+    ''' Aria2pFD class
+    This class support to use aria2p as downloader.
+    (Aria2p, a command-line tool and Python library to interact with an aria2c daemon process
+    through JSON-RPC.)
+    It can help you to get download progress more easily.
+    To use aria2p as downloader, you need to install aria2c and aria2p, aria2p can download with pip.
+    Then run aria2c in the background and enable with the --enable-rpc option.
+    '''
+    try:
+        import aria2p
+        __avail = True
+    except ImportError:
+        __avail = False
+
+    @classmethod
+    def available(cls):
+        return cls.__avail
+
+    def _call_downloader(self, tmpfilename, info_dict):
+        aria2 = self.aria2p.API(
+            self.aria2p.Client(
+                host='http://localhost',
+                port=6800,
+                secret=''
+            )
+        )
+
+        options = {
+            'min-split-size': '1M',
+            'max-connection-per-server': 4,
+            'auto-file-renaming': 'false',
+        }
+        options['dir'] = os.path.dirname(tmpfilename) or os.path.abspath('.')
+        options['out'] = os.path.basename(tmpfilename)
+        options['header'] = []
+        for key, val in info_dict['http_headers'].items():
+            options['header'].append('{0}: {1}'.format(key, val))
+        download = aria2.add_uris([info_dict['url']], options)
+        status = {
+            'status': 'downloading',
+            'tmpfilename': tmpfilename,
+        }
+        started = time.time()
+        while download.status in ['active', 'waiting']:
+            download = aria2.get_download(download.gid)
+            status.update({
+                'downloaded_bytes': download.completed_length,
+                'total_bytes': download.total_length,
+                'elapsed': time.time() - started,
+                'eta': download.eta.total_seconds(),
+                'speed': download.download_speed,
+            })
+            self._hook_progress(status)
+            time.sleep(.5)
+        return download.status != 'complete'
+
+
 class HttpieFD(ExternalFD):
     @classmethod
     def available(cls):

From 822f19f05d0ab1a4a945a85f691f2079f7cb3bbb Mon Sep 17 00:00:00 2001
From: fonkap <fonk666@gmail.com>
Date: Sat, 11 Feb 2023 03:37:45 +0100
Subject: [PATCH 600/860] [FileMoonIE] Add extractor for filemoon.sx (#31515)

---------

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/filemoon.py   | 43 ++++++++++++++++++++++++++++++
 2 files changed, 44 insertions(+)
 create mode 100644 youtube_dl/extractor/filemoon.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index dfaef0cc3..f63a2e030 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -376,6 +376,7 @@ from .fc2 import (
     FC2EmbedIE,
 )
 from .fczenit import FczenitIE
+from .filemoon import FileMoonIE
 from .fifa import FifaIE
 from .filmon import (
     FilmOnIE,
diff --git a/youtube_dl/extractor/filemoon.py b/youtube_dl/extractor/filemoon.py
new file mode 100644
index 000000000..654df9b69
--- /dev/null
+++ b/youtube_dl/extractor/filemoon.py
@@ -0,0 +1,43 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..utils import (
+    decode_packed_codes,
+    js_to_json,
+)
+
+
+class FileMoonIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?filemoon\.sx/./(?P<id>\w+)'
+    _TEST = {
+        'url': 'https://filemoon.sx/e/dw40rxrzruqz',
+        'md5': '5a713742f57ac4aef29b74733e8dda01',
+        'info_dict': {
+            'id': 'dw40rxrzruqz',
+            'title': 'dw40rxrzruqz',
+            'ext': 'mp4'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+
+        webpage = self._download_webpage(url, video_id)
+        matches = re.findall(r'(?s)(eval.*?)</script>', webpage)
+        packed = matches[-1]
+        unpacked = decode_packed_codes(packed)
+        jwplayer_sources = self._parse_json(
+            self._search_regex(
+                r'(?s)player\s*\.\s*setup\s*\(\s*\{\s*sources\s*:\s*(.*?])', unpacked, 'jwplayer sources'),
+            video_id, transform_source=js_to_json)
+
+        formats = self._parse_jwplayer_formats(jwplayer_sources, video_id)
+
+        return {
+            'id': video_id,
+            'title': self._generic_title(url) or video_id,
+            'formats': formats
+        }

From de48105dd870e353af468bfb8d49b14d9894e649 Mon Sep 17 00:00:00 2001
From: fonkap <fonk666@gmail.com>
Date: Sat, 11 Feb 2023 03:47:43 +0100
Subject: [PATCH 601/860] [KommunetvIE] Add extractor for kommunetv.no (#31516)

* Add extractor for kommunetv.no
* Using utils.update_url instead of regex

---------

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/kommunetv.py  | 35 ++++++++++++++++++++++++++++++
 2 files changed, 36 insertions(+)
 create mode 100644 youtube_dl/extractor/kommunetv.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index f63a2e030..d8428f46f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -557,6 +557,7 @@ from .khanacademy import (
 from .kickstarter import KickStarterIE
 from .kinja import KinjaEmbedIE
 from .kinopoisk import KinoPoiskIE
+from .kommunetv import KommunetvIE
 from .konserthusetplay import KonserthusetPlayIE
 from .krasview import KrasViewIE
 from .kth import KTHIE
diff --git a/youtube_dl/extractor/kommunetv.py b/youtube_dl/extractor/kommunetv.py
new file mode 100644
index 000000000..91d06a74f
--- /dev/null
+++ b/youtube_dl/extractor/kommunetv.py
@@ -0,0 +1,35 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import update_url
+
+
+class KommunetvIE(InfoExtractor):
+    _VALID_URL = r'https://(\w+).kommunetv.no/archive/(?P<id>\w+)'
+    _TEST = {
+        'url': 'https://oslo.kommunetv.no/archive/921',
+        'md5': '5f102be308ee759be1e12b63d5da4bbc',
+        'info_dict': {
+            'id': '921',
+            'title': 'Bystyremøte',
+            'ext': 'mp4'
+        }
+    }
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        headers = {
+            'Accept': 'application/json'
+        }
+        data = self._download_json('https://oslo.kommunetv.no/api/streams?streamType=1&id=%s' % video_id, video_id, headers=headers)
+        title = data['stream']['title']
+        file = data['playlist'][0]['playlist'][0]['file']
+        url = update_url(file, query=None, fragment=None)
+        formats = self._extract_m3u8_formats(url, video_id, ext='mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
+        self._sort_formats(formats)
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': title
+        }

From 6f8c2635a573c84ef66c02f73b4aeff1cc36ae4e Mon Sep 17 00:00:00 2001
From: fonkap <fonk666@gmail.com>
Date: Sat, 11 Feb 2023 03:54:45 +0100
Subject: [PATCH 602/860] [StreamsbIE] Add extractor for streamsb.com
 (viewsb.com) (#31517)

* Add extractor for streamsb.com (viewsb.com)

* make data url using app.js version

---------

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/streamsb.py   | 61 ++++++++++++++++++++++++++++++
 2 files changed, 62 insertions(+)
 create mode 100644 youtube_dl/extractor/streamsb.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d8428f46f..3a87f9e33 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1206,6 +1206,7 @@ from .storyfire import (
 from .streamable import StreamableIE
 from .streamcloud import StreamcloudIE
 from .streamcz import StreamCZIE
+from .streamsb import StreamsbIE
 from .streetvoice import StreetVoiceIE
 from .stretchinternet import StretchInternetIE
 from .stv import STVPlayerIE
diff --git a/youtube_dl/extractor/streamsb.py b/youtube_dl/extractor/streamsb.py
new file mode 100644
index 000000000..bffcb3de1
--- /dev/null
+++ b/youtube_dl/extractor/streamsb.py
@@ -0,0 +1,61 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import binascii
+import random
+import re
+import string
+
+from .common import InfoExtractor
+from ..utils import urljoin, url_basename
+
+
+def to_ascii_hex(str1):
+    return binascii.hexlify(str1.encode('utf-8')).decode('ascii')
+
+
+def generate_random_string(length):
+    return ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(length))
+
+
+class StreamsbIE(InfoExtractor):
+    _DOMAINS = ('viewsb.com', )
+    _VALID_URL = r'https://(?P<domain>%s)/(?P<id>.+)' % '|'.join(_DOMAINS)
+    _TEST = {
+        'url': 'https://viewsb.com/dxfvlu4qanjx',
+        'md5': '488d111a63415369bf90ea83adc8a325',
+        'info_dict': {
+            'id': 'dxfvlu4qanjx',
+            'ext': 'mp4',
+            'title': 'Sintel'
+        }
+    }
+
+    def _real_extract(self, url):
+        domain, video_id = re.match(self._VALID_URL, url).group('domain', 'id')
+        webpage = self._download_webpage(url, video_id)
+
+        iframe_rel_url = self._search_regex(r'''(?i)<iframe\b[^>]+\bsrc\s*=\s*('|")(?P<path>/.*\.html)\1''', webpage, 'iframe', group='path')
+        iframe_url = urljoin('https://' + domain, iframe_rel_url)
+
+        iframe_data = self._download_webpage(iframe_url, video_id)
+        app_version = self._search_regex(r'''<script\b[^>]+\bsrc\s*=\s*["|'].*/app\.min\.(\d+)\.js''', iframe_data, 'app version', fatal=False) or '50'
+
+        video_code = url_basename(iframe_url).rsplit('.')[0]
+
+        length = 12
+        req = '||'.join((generate_random_string(length), video_code, generate_random_string(length), 'streamsb'))
+        ereq = 'https://{0}/sources{1}/{2}'.format(domain, app_version, to_ascii_hex(req))
+
+        video_data = self._download_webpage(ereq, video_id, headers={
+            'Referer': iframe_url,
+            'watchsb': 'sbstream',
+        })
+        player_data = self._parse_json(video_data, video_id)
+        title = player_data['stream_data']['title']
+        formats = self._extract_m3u8_formats(player_data['stream_data']['file'], video_id, ext='mp4', entry_protocol='m3u8_native', m3u8_id='hls', fatal=False)
+        return {
+            'id': video_id,
+            'formats': formats,
+            'title': title,
+        }

From 42b098dd79e91295376ca98f394876555481a3eb Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 14 Feb 2023 02:47:09 +0000
Subject: [PATCH 603/860] [InfoExtractor] Handle unquoted values in OpenGraph
 searches

---
 test/test_InfoExtractor.py     | 2 ++
 youtube_dl/extractor/common.py | 2 +-
 2 files changed, 3 insertions(+), 1 deletion(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index dd69a681b..4db5c93f1 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -62,6 +62,7 @@ class TestInfoExtractor(unittest.TestCase):
             <meta name="og:test1" content='foo > < bar'/>
             <meta name="og:test2" content="foo >//< bar"/>
             <meta property=og-test3 content='Ill-formatted opengraph'/>
+            <meta property=og:test4 content=unquoted-value/>
             '''
         self.assertEqual(ie._og_search_title(html), 'Foo')
         self.assertEqual(ie._og_search_description(html), 'Some video\'s description ')
@@ -74,6 +75,7 @@ class TestInfoExtractor(unittest.TestCase):
         self.assertEqual(ie._og_search_property(('test0', 'test1'), html), 'foo > < bar')
         self.assertRaises(RegexNotFoundError, ie._og_search_property, 'test0', html, None, fatal=True)
         self.assertRaises(RegexNotFoundError, ie._og_search_property, ('test0', 'test00'), html, None, fatal=True)
+        self.assertEqual(ie._og_search_property('test4', html), 'unquoted-value')
 
     def test_html_search_meta(self):
         ie = self.ie
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index a0a796d7b..7244e5df6 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1087,7 +1087,7 @@ class InfoExtractor(object):
     # Helper functions for extracting OpenGraph info
     @staticmethod
     def _og_regexes(prop):
-        content_re = r'content=(?:"([^"]+?)"|\'([^\']+?)\'|\s*([^\s"\'=<>`]+?))'
+        content_re = r'content=(?:"([^"]+?)"|\'([^\']+?)\'|\s*([^\s"\'=<>`]+?)(?=\s|/?>))'
         property_re = (r'(?:name|property)=(?:\'og[:-]%(prop)s\'|"og[:-]%(prop)s"|\s*og[:-]%(prop)s\b)'
                        % {'prop': re.escape(prop)})
         template = r'<meta[^>]+?%s[^>]+?%s'

From dd9aa74beefc179f943051c4e19eecad87ab1124 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 14 Feb 2023 16:33:01 +0000
Subject: [PATCH 604/860] [test] Avoid name TestIE which causes a pytest
 warning

See: https://github.com/yt-dlp/yt-dlp/commit/060ac76257a8c1f7370a8a571821c1d73377701f
---
 test/test_InfoExtractor.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 4db5c93f1..6d25441db 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -35,13 +35,13 @@ class InfoExtractorTestRequestHandler(compat_http_server.BaseHTTPRequestHandler)
             assert False
 
 
-class TestIE(InfoExtractor):
+class DummyIE(InfoExtractor):
     pass
 
 
 class TestInfoExtractor(unittest.TestCase):
     def setUp(self):
-        self.ie = TestIE(FakeYDL())
+        self.ie = DummyIE(FakeYDL())
 
     def test_ie_key(self):
         self.assertEqual(get_info_extractor(YoutubeIE.ie_key()), YoutubeIE)

From 2dd6c6edd8e0fc5e45865b8e6d865e35147de772 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 17 Feb 2023 11:16:54 +0000
Subject: [PATCH 605/860] [YouTube] Avoid crash if uploader_id extraction fails

See #31530.
---
 youtube_dl/extractor/youtube.py | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ba0f5c8b6..66b0257df 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2122,7 +2122,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 microformat.get('uploadDate')
                 or search_meta('uploadDate')),
             'uploader': video_details['author'],
-            'uploader_id': self._search_regex(r'/(?:channel|user)/([^/?&#]+)', owner_profile_url, 'uploader id') if owner_profile_url else None,
+            'uploader_id': self._search_regex(
+                r'/(?:channel|user)/([^/?&#]+)', owner_profile_url,
+                'uploader id', fatal=False) if owner_profile_url else None,
             'uploader_url': owner_profile_url,
             'channel_id': channel_id,
             'channel_url': 'https://www.youtube.com/channel/' + channel_id if channel_id else None,

From 57802e632f5a741df6fd9b30a455c32632944489 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 19 Feb 2023 13:47:49 +0000
Subject: [PATCH 606/860] [jsinterp] Fix dict comprehension for Py2.6

Resolves #31600
---
 youtube_dl/jsinterp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 60fa2b1b9..a3bc42a61 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -262,7 +262,7 @@ class JSInterpreter(object):
         if not expr:
             return
         # collections.Counter() is ~10% slower in both 2.7 and 3.9
-        counters = {k: 0 for k in _MATCHING_PARENS.values()}
+        counters = dict((k, 0) for k in _MATCHING_PARENS.values())
         start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
         in_quote, escaping, skipping = None, False, 0
         after_op, in_regex_char_group, skip_re = True, False, 0

From 6067451e432fb65d487a8a67bb5cff52efb9ccf4 Mon Sep 17 00:00:00 2001
From: df <fieldhouse@gmx.net>
Date: Mon, 20 Feb 2023 01:41:46 +0000
Subject: [PATCH 607/860] [Vimeo] Fix e19ec52 for tween-age Pythons

* a check in older Pythons in the 2.7 and earlier, 3.3, 3.4 series caused "sre_constants.error: nothing to repeat"
* satisfy the check by avoiding nested qualifiers that can match empty string

Resolves #31597
---
 youtube_dl/extractor/vimeo.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 7f2731d83..8e1a805f6 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -286,7 +286,7 @@ class VimeoIE(VimeoBaseInfoExtractor):
                              /(?!videos|likes)[^/?#]+/?|
                              (?(q)|/(?P<unlisted_hash>[\da-f]{10}))?
                          )
-                         (?:(?(q)[&]|(?(u)|/?)[?]).*?)?(?:[#].*)?$
+                         (?:(?(q)[&]|(?(u)|/?)[?]).+?)?(?:[#].*)?$
                  '''
     IE_NAME = 'vimeo'
     _TESTS = [

From 1d3751c3fe50b203d3e2bff71d866c8c500f8288 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 1 Jun 2021 18:05:41 +0530
Subject: [PATCH 608/860] Escape URLs in `sanitized_Request`, not
 `sanitize_url` d2558234cf5dd12d6896eed5427b7dcdb3ab7b5a added escaping of
 URLs while sanitizing. However, `sanitize_url` may not always receive an
 actual URL. Eg: When using `youtube-dl "search query" --default-search
 ytsearch`, `search query` gets escaped to `search%20query` before being
 prefixed with `ytsearch:` which is not the intended behavior. So the escaping
 is moved to `sanitized_Request` instead.

---
 test/test_utils.py              |  1 +
 youtube_dl/extractor/generic.py | 19 +++++++++++++++++++
 youtube_dl/utils.py             |  4 ++--
 3 files changed, 22 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 9d364c863..ea2b96ed2 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -250,6 +250,7 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(sanitize_url('httpss://foo.bar'), 'https://foo.bar')
         self.assertEqual(sanitize_url('rmtps://foo.bar'), 'rtmps://foo.bar')
         self.assertEqual(sanitize_url('https://foo.bar'), 'https://foo.bar')
+        self.assertEqual(sanitize_url('foo bar'), 'foo bar')
 
     def test_expand_path(self):
         def env(var):
diff --git a/youtube_dl/extractor/generic.py b/youtube_dl/extractor/generic.py
index 0e473e952..b01900afa 100644
--- a/youtube_dl/extractor/generic.py
+++ b/youtube_dl/extractor/generic.py
@@ -2320,6 +2320,25 @@ class GenericIE(InfoExtractor):
                 'height': 720,
                 'age_limit': 18,
             },
+        }, {
+            # would like to use the yt-dl test video but searching for
+            # '"\'/\\ä↭𝕐' fails, so using an old vid from YouTube Korea
+            'note': 'Test default search',
+            'url': 'Shorts로 허락 필요없이 놀자! (BTS편)',
+            'info_dict': {
+                'id': 'usDGO4Zb-dc',
+                'ext': 'mp4',
+                'title': 'YouTube Shorts로 허락 필요없이 놀자! (BTS편)',
+                'description': 'md5:96e31607eba81ab441567b5e289f4716',
+                'upload_date': '20211107',
+                'uploader': 'YouTube Korea',
+                'location': '대한민국',
+            },
+            'params': {
+                'default_search': 'ytsearch',
+                'skip_download': True,
+            },
+            'expected_warnings': ['uploader id'],
         },
     ]
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 4edbfa27b..761edcd49 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2176,11 +2176,11 @@ def sanitize_url(url):
     for mistake, fixup in COMMON_TYPOS:
         if re.match(mistake, url):
             return re.sub(mistake, fixup, url)
-    return escape_url(url)
+    return url
 
 
 def sanitized_Request(url, *args, **kwargs):
-    return compat_urllib_request.Request(sanitize_url(url), *args, **kwargs)
+    return compat_urllib_request.Request(escape_url(sanitize_url(url)), *args, **kwargs)
 
 
 def expand_path(s):

From e67e52a8f8fd7e76253e416da76570af8da200d0 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 24 Feb 2023 02:32:40 +0000
Subject: [PATCH 609/860] [test] Support test-case with volatile ID (eg live
 show)

Signalled by regexp ID value, eg: `'id': r're:[\da-zA-Z_-]{8,}'`
---
 test/test_download.py | 9 +++++++++
 1 file changed, 9 insertions(+)

diff --git a/test/test_download.py b/test/test_download.py
index 19936969f..d50008307 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -148,6 +148,7 @@ def generator(test_case, tname):
                 try_rm(tc_filename)
                 try_rm(tc_filename + '.part')
                 try_rm(os.path.splitext(tc_filename)[0] + '.info.json')
+
         try_rm_tcs_files()
         try:
             try_num = 1
@@ -213,7 +214,15 @@ def generator(test_case, tname):
                 # First, check test cases' data against extracted data alone
                 expect_info_dict(self, tc_res_dict, tc.get('info_dict', {}))
                 # Now, check downloaded file consistency
+                # support test-case with volatile ID, signalled by regexp value
+                if tc.get('info_dict', {}).get('id', '').startswith('re:'):
+                    test_id = tc['info_dict']['id']
+                    tc['info_dict']['id'] = tc_res_dict['id']
+                else:
+                    test_id = None
                 tc_filename = get_tc_filename(tc)
+                if test_id:
+                    tc['info_dict']['id'] = test_id
                 if not test_case.get('params', {}).get('skip_download', False):
                     self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename)
                     self.assertTrue(tc_filename in finished_hook_called)

From f7ce98a21e15cb094c772e9082796d009c61578b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 24 Feb 2023 02:48:37 +0000
Subject: [PATCH 610/860] [YouTube] Support @owner format in uploader_id etc

* implement https://github.com/ytdl-org/youtube-dl/issues/31530#issuecomment-1435734719
* update affected tests
* misc clean-ups
---
 youtube_dl/extractor/youtube.py | 319 +++++++++++++++++++-------------
 1 file changed, 194 insertions(+), 125 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 66b0257df..4246d84f9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -31,6 +31,7 @@ from ..utils import (
     get_element_by_attribute,
     int_or_none,
     js_to_json,
+    merge_dicts,
     mimetype2ext,
     parse_codecs,
     parse_duration,
@@ -400,6 +401,62 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                 break
             data['continuation'] = token
 
+    @staticmethod
+    def _owner_endpoints_path():
+        return [
+            Ellipsis,
+            lambda k, _: k.endswith('SecondaryInfoRenderer'),
+            ('owner', 'videoOwner'), 'videoOwnerRenderer', 'title',
+            'runs', Ellipsis]
+
+    def _extract_channel_id(self, webpage, videodetails={}, metadata={}, renderers=[]):
+        channel_id = None
+        if any((videodetails, metadata, renderers)):
+            channel_id = (
+                traverse_obj(videodetails, 'channelId')
+                or traverse_obj(metadata, 'externalChannelId', 'externalId')
+                or traverse_obj(renderers,
+                                self._owner_endpoints_path() + [
+                                    'navigationEndpoint', 'browseEndpoint', 'browseId'],
+                                get_all=False)
+            )
+        return channel_id or self._html_search_meta(
+            'channelId', webpage, 'channel id', default=None)
+
+    def _extract_author_var(self, webpage, var_name,
+                            videodetails={}, metadata={}, renderers=[]):
+        result = None
+        paths = {
+            #       (HTML, videodetails, metadata, renderers)
+            'name': ('content', 'author', (('ownerChannelName', None), 'title'), ['text']),
+            'url': ('href', 'ownerProfileUrl', 'vanityChannelUrl',
+                    ['navigationEndpoint', 'browseEndpoint', 'canonicalBaseUrl'])
+        }
+        if any((videodetails, metadata, renderers)):
+            result = (
+                traverse_obj(videodetails, paths[var_name][1], get_all=False)
+                or traverse_obj(metadata, paths[var_name][2], get_all=False)
+                or traverse_obj(renderers,
+                                self._owner_endpoints_path() + paths[var_name][3],
+                                get_all=False)
+            )
+        return result or traverse_obj(
+            extract_attributes(self._search_regex(
+                r'''(?s)(<link\b[^>]+\bitemprop\s*=\s*("|')%s\2[^>]*>)'''
+                % re.escape(var_name),
+                get_element_by_attribute('itemprop', 'author', webpage) or '',
+                'author link', default='')),
+            paths[var_name][0])
+
+    @staticmethod
+    def _yt_urljoin(url_or_path):
+        return urljoin('https://www.youtube.com', url_or_path)
+
+    def _extract_uploader_id(self, uploader_url):
+        return self._search_regex(
+            r'/(?:(?:channel|user)/|(?=@))([^/?&#]+)', uploader_url or '',
+            'uploader id', default=None)
+
 
 class YoutubeIE(YoutubeBaseInfoExtractor):
     IE_DESC = 'YouTube.com'
@@ -516,8 +573,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
                 'uploader': 'Philipp Hagemeister',
-                'uploader_id': 'phihag',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
+                'uploader_id': '@PhilippHagemeister',
+                'uploader_url': r're:https?://(?:www\.)?youtube\.com/@PhilippHagemeister',
                 'channel': 'Philipp Hagemeister',
                 'channel_id': 'UCLqxVugv74EIW3VWh2NOa3Q',
                 'channel_url': r're:https?://(?:www\.)?youtube\.com/channel/UCLqxVugv74EIW3VWh2NOa3Q',
@@ -557,8 +614,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'title': 'youtube-dl test video "\'/\\ä↭𝕐',
                 'uploader': 'Philipp Hagemeister',
-                'uploader_id': 'phihag',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/phihag',
+                'uploader_id': '@PhilippHagemeister',
+                'uploader_url': r're:https?://(?:www\.)?youtube\.com/@PhilippHagemeister',
                 'upload_date': '20121002',
                 'description': 'test chars:  "\'/\\ä↭𝕐\ntest URL: https://github.com/rg3/youtube-dl/issues/1892\n\nThis is a test video for youtube-dl.\n\nFor more information, contact phihag@phihag.de .',
                 'categories': ['Science & Technology'],
@@ -588,7 +645,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'youtube_include_dash_manifest': True,
                 'format': '141',
             },
-            'skip': 'format 141 not served anymore',
+            'skip': 'format 141 not served any more',
         },
         # DASH manifest with encrypted signature
         {
@@ -600,7 +657,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:8f5e2b82460520b619ccac1f509d43bf',
                 'duration': 244,
                 'uploader': 'AfrojackVEVO',
-                'uploader_id': 'AfrojackVEVO',
+                'uploader_id': '@AfrojackVEVO',
                 'upload_date': '20131011',
                 'abr': 129.495,
             },
@@ -618,8 +675,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'duration': 219,
                 'upload_date': '20100909',
                 'uploader': 'Amazing Atheist',
-                'uploader_id': 'TheAmazingAtheist',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/TheAmazingAtheist',
+                'uploader_id': '@theamazingatheist',
+                'uploader_url': r're:https?://(?:www\.)?youtube\.com/@theamazingatheist',
                 'title': 'Burning Everyone\'s Koran',
                 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms \r\n\r\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
             }
@@ -635,8 +692,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': r're:(?s).{100,}About the Game\n.*?The Witcher 3: Wild Hunt.{100,}',
                 'duration': 142,
                 'uploader': 'The Witcher',
-                'uploader_id': 'WitcherGame',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/WitcherGame',
+                'uploader_id': '@thewitcher',
+                'uploader_url': r're:https?://(?:www\.)?youtube\.com/@thewitcher',
                 'upload_date': '20140605',
                 'thumbnail': 'https://i.ytimg.com/vi/HtVdAasjOgU/maxresdefault.jpg',
                 'age_limit': 18,
@@ -659,7 +716,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:bf77e03fcae5529475e500129b05668a',
                 'duration': 177,
                 'uploader': 'FlyingKitty',
-                'uploader_id': 'FlyingKitty900',
+                'uploader_id': '@FlyingKitty900',
                 'upload_date': '20200408',
                 'thumbnail': 'https://i.ytimg.com/vi/HsUATh_Nc2U/maxresdefault.jpg',
                 'age_limit': 18,
@@ -682,7 +739,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:17eccca93a786d51bc67646756894066',
                 'duration': 106,
                 'uploader': 'Projekt Melody',
-                'uploader_id': 'UC1yoRdFoFJaCY-AGfD9W0wQ',
+                'uploader_id': '@ProjektMelody',
                 'upload_date': '20191227',
                 'age_limit': 18,
                 'thumbnail': 'https://i.ytimg.com/vi/Tq92D6wQ1mg/sddefault.jpg',
@@ -704,10 +761,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'OOMPH! - Such Mich Find Mich (Lyrics)',
                 'description': 'Fan Video. Music & Lyrics by OOMPH!.',
                 'duration': 210,
-                'uploader': 'Herr Lurik',
-                'uploader_id': 'st3in234',
                 'upload_date': '20130730',
-                'uploader_url': 'http://www.youtube.com/user/st3in234',
+                'uploader': 'Herr Lurik',
+                'uploader_id': '@HerrLurik',
+                'uploader_url': 'http://www.youtube.com/@HerrLurik',
                 'age_limit': 0,
                 'thumbnail': 'https://i.ytimg.com/vi/MeJVWBSsPAY/hqdefault.jpg',
                 'tags': ['oomph', 'such mich find mich', 'lyrics', 'german industrial', 'musica industrial'],
@@ -740,8 +797,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'ext': 'mp4',
                 'duration': 266,
                 'upload_date': '20100430',
-                'uploader_id': 'deadmau5',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/deadmau5',
+                'uploader_id': '@deadmau5',
+                'uploader_url': r're:https?://(?:www\.)?youtube\.com/@deadmau5',
                 'creator': 'deadmau5',
                 'description': 'md5:6cbcd3a92ce1bc676fc4d6ab4ace2336',
                 'uploader': 'deadmau5',
@@ -762,8 +819,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': r're:(?s)(?:.+\s)?HO09  - Women -  GER-AUS - Hockey - 31 July 2012 - London 2012 Olympic Games\s*',
                 'duration': 6085,
                 'upload_date': '20150827',
-                'uploader_id': 'olympic',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/olympic',
+                'uploader_id': '@Olympics',
+                'uploader_url': r're:https?://(?:www\.)?youtube\.com/@Olympics',
                 'uploader': r're:Olympics?',
                 'age_limit': 0,
                 'thumbnail': 'https://i.ytimg.com/vi/lqQg6PlCWgI/maxresdefault.jpg',
@@ -785,8 +842,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'stretched_ratio': 16 / 9.,
                 'duration': 85,
                 'upload_date': '20110310',
-                'uploader_id': 'AllenMeow',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/AllenMeow',
+                'uploader_id': '@AllenMeow',
+                'uploader_url': r're:https?://(?:www\.)?youtube\.com/@AllenMeow',
                 'description': 'made by Wacom from Korea | 字幕&加油添醋 by TY\'s Allen | 感謝heylisa00cavey1001同學熱情提供梗及翻譯',
                 'uploader': '孫ᄋᄅ',
                 'title': '[A-made] 變態妍字幕版 太妍 我就是這樣的人',
@@ -824,7 +881,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader': 'dorappi2000',
                 'formats': 'mincount:31',
             },
-            'skip': 'not actual anymore',
+            'skip': 'not actual any more',
         },
         # DASH manifest with segment_list
         {
@@ -905,6 +962,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'params': {
                 'skip_download': True,
             },
+            'skip': 'Not multifeed any more',
         },
         {
             # Multifeed video with comma in title (see https://github.com/ytdl-org/youtube-dl/issues/8536)
@@ -914,7 +972,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'DevConf.cz 2016 Day 2 Workshops 1 14:00 - 15:30',
             },
             'playlist_count': 2,
-            'skip': 'Not multifeed anymore',
+            'skip': 'Not multifeed any more',
         },
         {
             'url': 'https://vid.plus/FlRa-iH7PGw',
@@ -938,8 +996,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:8085699c11dc3f597ce0410b0dcbb34a',
                 'duration': 133,
                 'upload_date': '20151119',
-                'uploader_id': 'IronSoulElf',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/IronSoulElf',
+                'uploader_id': '@IronSoulElf',
+                'uploader_url': r're:https?://(?:www\.)?youtube\.com/@IronSoulElf',
                 'uploader': 'IronSoulElf',
                 'creator': r're:Todd Haberman[;,]\s+Daniel Law Heath and Aaron Kaplan',
                 'track': 'Dark Walk',
@@ -987,8 +1045,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:a677553cf0840649b731a3024aeff4cc',
                 'duration': 721,
                 'upload_date': '20150127',
-                'uploader_id': 'BerkmanCenter',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/BerkmanCenter',
+                'uploader_id': '@BKCHarvard',
+                'uploader_url': r're:https?://(?:www\.)?youtube\.com/@BKCHarvard',
                 'uploader': 'The Berkman Klein Center for Internet & Society',
                 'license': 'Creative Commons Attribution license (reuse allowed)',
             },
@@ -1007,8 +1065,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'duration': 4060,
                 'upload_date': '20151119',
                 'uploader': 'Bernie Sanders',
-                'uploader_id': 'UCH1dpzjCEiGAt8CXkryhkZg',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UCH1dpzjCEiGAt8CXkryhkZg',
+                'uploader_id': '@BernieSanders',
+                'uploader_url': r're:https?://(?:www\.)?youtube\.com/@BernieSanders',
                 'license': 'Creative Commons Attribution license (reuse allowed)',
             },
             'params': {
@@ -1054,8 +1112,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'duration': 2085,
                 'upload_date': '20170118',
                 'uploader': 'Vsauce',
-                'uploader_id': 'Vsauce',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/Vsauce',
+                'uploader_id': '@Vsauce',
+                'uploader_url': r're:https?://(?:www\.)?youtube\.com/@Vsauce',
                 'series': 'Mind Field',
                 'season_number': 1,
                 'episode_number': 1,
@@ -1134,7 +1192,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'skip_download': True,
                 'youtube_include_dash_manifest': False,
             },
-            'skip': 'not actual anymore',
+            'skip': 'not actual any more',
         },
         {
             # Youtube Music Auto-generated description
@@ -1191,8 +1249,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'IMG 3456',
                 'description': '',
                 'upload_date': '20170613',
-                'uploader_id': 'ElevageOrVert',
                 'uploader': 'ElevageOrVert',
+                'uploader_id': '@ElevageOrVert',
             },
             'params': {
                 'skip_download': True,
@@ -1210,8 +1268,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'Part 77   Sort a list of simple types in c#',
                 'description': 'md5:b8746fa52e10cdbf47997903f13b20dc',
                 'upload_date': '20130831',
-                'uploader_id': 'kudvenkat',
                 'uploader': 'kudvenkat',
+                'uploader_id': '@Csharp-video-tutorialsBlogspot',
             },
             'params': {
                 'skip_download': True,
@@ -1263,8 +1321,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'description': 'md5:ea770e474b7cd6722b4c95b833c03630',
                 'upload_date': '20201120',
                 'uploader': 'Walk around Japan',
-                'uploader_id': 'UC3o_t8PzBmXf5S9b7GLx1Mw',
-                'uploader_url': r're:https?://(?:www\.)?youtube\.com/channel/UC3o_t8PzBmXf5S9b7GLx1Mw',
+                'uploader_id': '@walkaroundjapan7124',
+                'uploader_url': r're:https?://(?:www\.)?youtube\.com/@walkaroundjapan7124',
             },
             'params': {
                 'skip_download': True,
@@ -1276,11 +1334,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'info_dict': {
                 'id': '4L2J27mJ3Dc',
                 'ext': 'mp4',
+                'title': 'Midwest Squid Game #Shorts',
+                'description': 'md5:976512b8a29269b93bbd8a61edc45a6d',
                 'upload_date': '20211025',
                 'uploader': 'Charlie Berens',
-                'description': 'md5:976512b8a29269b93bbd8a61edc45a6d',
-                'uploader_id': 'fivedlrmilkshake',
-                'title': 'Midwest Squid Game #Shorts',
+                'uploader_id': '@CharlieBerens',
             },
             'params': {
                 'skip_download': True,
@@ -2088,25 +2146,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 thumbnails = [{'url': thumbnail}]
 
         category = microformat.get('category') or search_meta('genre')
-        channel_id = video_details.get('channelId') \
-            or microformat.get('externalChannelId') \
-            or search_meta('channelId')
+        channel_id = self._extract_channel_id(
+            webpage, videodetails=video_details, metadata=microformat)
         duration = int_or_none(
             video_details.get('lengthSeconds')
             or microformat.get('lengthSeconds')) \
             or parse_duration(search_meta('duration'))
         is_live = video_details.get('isLive')
 
-        def gen_owner_profile_url():
-            yield microformat.get('ownerProfileUrl')
-            yield extract_attributes(self._search_regex(
-                r'''(?s)(<link\b[^>]+\bitemprop\s*=\s*("|')url\2[^>]*>)''',
-                get_element_by_attribute('itemprop', 'author', webpage),
-                'owner_profile_url', default='')).get('href')
+        owner_profile_url = self._yt_urljoin(self._extract_author_var(
+            webpage, 'url', videodetails=video_details, metadata=microformat))
 
-        owner_profile_url = next(
-            (x for x in map(url_or_none, gen_owner_profile_url()) if x),
-            None)
+        uploader = self._extract_author_var(
+            webpage, 'name', videodetails=video_details, metadata=microformat)
 
         if not player_url:
             player_url = self._extract_player_url(webpage)
@@ -2121,13 +2173,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'upload_date': unified_strdate(
                 microformat.get('uploadDate')
                 or search_meta('uploadDate')),
-            'uploader': video_details['author'],
-            'uploader_id': self._search_regex(
-                r'/(?:channel|user)/([^/?&#]+)', owner_profile_url,
-                'uploader id', fatal=False) if owner_profile_url else None,
-            'uploader_url': owner_profile_url,
+            'uploader': uploader,
             'channel_id': channel_id,
-            'channel_url': 'https://www.youtube.com/channel/' + channel_id if channel_id else None,
             'duration': duration,
             'view_count': int_or_none(
                 video_details.get('viewCount')
@@ -2257,6 +2304,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 initial_data,
                 lambda x: x['contents']['twoColumnWatchNextResults']['results']['results']['contents'],
                 list) or []
+            if not info['channel_id']:
+                channel_id = self._extract_channel_id('', renderers=contents)
+            if not info['uploader']:
+                info['uploader'] = self._extract_author_var('', 'name', renderers=contents)
+            if not owner_profile_url:
+                owner_profile_url = self._yt_urljoin(self._extract_author_var('', 'url', renderers=contents))
+
             for content in contents:
                 vpir = content.get('videoPrimaryInfoRenderer')
                 if vpir:
@@ -2304,10 +2358,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         })
                 vsir = content.get('videoSecondaryInfoRenderer')
                 if vsir:
-                    info['channel'] = get_text(try_get(
-                        vsir,
-                        lambda x: x['owner']['videoOwnerRenderer']['title'],
-                        dict))
                     rows = try_get(
                         vsir,
                         lambda x: x['metadataRowContainer']['metadataRowContainerRenderer']['rows'],
@@ -2365,7 +2415,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         self.mark_watched(video_id, player_response)
 
-        return info
+        return merge_dicts(
+            info, {
+                'uploader_id': self._extract_uploader_id(owner_profile_url),
+                'uploader_url': owner_profile_url,
+                'channel_id': channel_id,
+                'channel_url': channel_id and self._yt_urljoin('/channel/' + channel_id),
+                'channel': info['uploader'],
+            })
 
 
 class YoutubeTabIE(YoutubeBaseInfoExtractor):
@@ -2394,6 +2451,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'description': 'Short clips from Super Cooper Sundays!',
             'id': 'UCKMA8kHZ8bPYpnMNaUSxfEQ',
             'title': 'Super Cooper Shorts - Shorts',
+            'uploader': 'Super Cooper Shorts',
+            'uploader_id': '@SuperCooperShorts',
         }
     }, {
         # Channel that does not have a Shorts tab. Test should just download videos on Home tab instead
@@ -2404,14 +2463,17 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'title': 'Emergency Awesome - Home',
         },
         'playlist_mincount': 5,
+        'skip': 'new test page needed to replace `Emergency Awesome - Shorts`',
     }, {
         # playlists, multipage
         'url': 'https://www.youtube.com/c/ИгорьКлейнер/playlists?view=1&flow=grid',
         'playlist_mincount': 94,
         'info_dict': {
             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
-            'title': 'Игорь Клейнер - Playlists',
+            'title': 'Igor Kleiner - Playlists',
             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
+            'uploader': 'Igor Kleiner',
+            'uploader_id': '@IgorDataScience',
         },
     }, {
         # playlists, multipage, different order
@@ -2419,8 +2481,10 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
         'playlist_mincount': 94,
         'info_dict': {
             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
-            'title': 'Игорь Клейнер - Playlists',
+            'title': 'Igor Kleiner - Playlists',
             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
+            'uploader': 'Igor Kleiner',
+            'uploader_id': '@IgorDataScience',
         },
     }, {
         # playlists, series
@@ -2430,6 +2494,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'id': 'UCYO_jab_esuFRV4b17AJtAw',
             'title': '3Blue1Brown - Playlists',
             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
+            'uploader': '3Blue1Brown',
+            'uploader_id': '@3blue1brown',
         },
     }, {
         # playlists, singlepage
@@ -2439,6 +2505,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'id': 'UCAEtajcuhQ6an9WEzY9LEMQ',
             'title': 'ThirstForScience - Playlists',
             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
+            'uploader': 'ThirstForScience',
+            'uploader_id': '@ThirstForScience',
         }
     }, {
         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
@@ -2447,20 +2515,22 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
         # basic, single video playlist
         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
         'info_dict': {
-            'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
-            'uploader': 'Sergey M.',
             'id': 'PL4lCao7KL_QFVb7Iudeipvc2BCavECqzc',
             'title': 'youtube-dl public playlist',
+            'uploader': 'Sergey M.',
+            'uploader_id': '@sergeym.6173',
+            'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
         },
         'playlist_count': 1,
     }, {
         # empty playlist
         'url': 'https://www.youtube.com/playlist?list=PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
         'info_dict': {
-            'uploader_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
-            'uploader': 'Sergey M.',
             'id': 'PL4lCao7KL_QFodcLWhDpGCYnngnHtQ-Xf',
             'title': 'youtube-dl empty playlist',
+            'uploader': 'Sergey M.',
+            'uploader_id': '@sergeym.6173',
+            'channel_id': 'UCmlqkdCBesrv2Lak1mF_MxA',
         },
         'playlist_count': 0,
     }, {
@@ -2470,6 +2540,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
             'title': 'lex will - Home',
             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
+            'uploader': 'lex will',
+            'uploader_id': '@lexwill718',
         },
         'playlist_mincount': 2,
     }, {
@@ -2479,6 +2551,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
             'title': 'lex will - Videos',
             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
+            'uploader': 'lex will',
+            'uploader_id': '@lexwill718',
         },
         'playlist_mincount': 975,
     }, {
@@ -2488,6 +2562,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
             'title': 'lex will - Videos',
             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
+            'uploader': 'lex will',
+            'uploader_id': '@lexwill718',
         },
         'playlist_mincount': 199,
     }, {
@@ -2497,6 +2573,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
             'title': 'lex will - Playlists',
             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
+            'uploader': 'lex will',
+            'uploader_id': '@lexwill718',
         },
         'playlist_mincount': 17,
     }, {
@@ -2506,6 +2584,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
             'title': 'lex will - Community',
             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
+            'uploader': 'lex will',
+            'uploader_id': '@lexwill718',
         },
         'playlist_mincount': 18,
     }, {
@@ -2515,8 +2595,10 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
             'title': 'lex will - Channels',
             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
+            'uploader': 'lex will',
+            'uploader_id': '@lexwill718',
         },
-        'playlist_mincount': 138,
+        'playlist_mincount': 75,
     }, {
         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
         'only_matching': True,
@@ -2533,7 +2615,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'title': '29C3: Not my department',
             'id': 'PLwP_SiAcdui0KVebT0mU9Apz359a4ubsC',
             'uploader': 'Christiaan008',
-            'uploader_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
+            'uploader_id': '@ChRiStIaAn008',
+            'channel_id': 'UCEPzS1rYsrkqzSLNp76nrcg',
         },
         'playlist_count': 96,
     }, {
@@ -2543,7 +2626,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'title': 'Uploads from Cauchemar',
             'id': 'UUBABnxM4Ar9ten8Mdjj1j0Q',
             'uploader': 'Cauchemar',
-            'uploader_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
+            'uploader_id': '@Cauchemar89',
+            'channel_id': 'UCBABnxM4Ar9ten8Mdjj1j0Q',
         },
         'playlist_mincount': 1123,
     }, {
@@ -2557,7 +2641,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'title': 'Uploads from Interstellar Movie',
             'id': 'UUXw-G3eDE9trcvY2sBMM_aA',
             'uploader': 'Interstellar Movie',
-            'uploader_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
+            'uploader_id': '@InterstellarMovie',
+            'channel_id': 'UCXw-G3eDE9trcvY2sBMM_aA',
         },
         'playlist_mincount': 21,
     }, {
@@ -2566,8 +2651,9 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
         'info_dict': {
             'title': 'Data Analysis with Dr Mike Pound',
             'id': 'PLzH6n4zXuckpfMu_4Ff8E7Z1behQks5ba',
-            'uploader_id': 'UC9-y-6csu5WGm29I7JiwpnA',
             'uploader': 'Computerphile',
+            'uploader_id': '@Computerphile',
+            'channel_id': 'UC9-y-6csu5WGm29I7JiwpnA',
         },
         'playlist_mincount': 11,
     }, {
@@ -2605,14 +2691,14 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
     }, {
         'url': 'https://www.youtube.com/channel/UCoMdktPbSTixAyNGwb-UYkQ/live',
         'info_dict': {
-            'id': '9Auq9mYxFEE',
+            'id': r're:[\da-zA-Z_-]{8,}',
             'ext': 'mp4',
-            'title': 'Watch Sky News live',
+            'title': r're:(?s)[A-Z].{20,}',
             'uploader': 'Sky News',
-            'uploader_id': 'skynews',
-            'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/skynews',
-            'upload_date': '20191102',
-            'description': 'md5:78de4e1c2359d0ea3ed829678e38b662',
+            'uploader_id': '@SkyNews',
+            'uploader_url': r're:https?://(?:www\.)?youtube\.com/@SkyNews',
+            'upload_date': r're:\d{8}',
+            'description': r're:(?s)(?:.*\n)+SUBSCRIBE to our YouTube channel for more videos: http://www\.youtube\.com/skynews *\n.*',
             'categories': ['News & Politics'],
             'tags': list,
             'like_count': int,
@@ -2701,34 +2787,22 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
     }, {
         'note': 'Search tab',
         'url': 'https://www.youtube.com/c/3blue1brown/search?query=linear%20algebra',
-        'playlist_mincount': 40,
+        'playlist_mincount': 20,
         'info_dict': {
             'id': 'UCYO_jab_esuFRV4b17AJtAw',
             'title': '3Blue1Brown - Search - linear algebra',
             'description': 'md5:e1384e8a133307dd10edee76e875d62f',
             'uploader': '3Blue1Brown',
-            'uploader_id': 'UCYO_jab_esuFRV4b17AJtAw',
+            'uploader_id': '@3blue1brown',
+            'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
         }
     }]
 
     @classmethod
     def suitable(cls, url):
-        return False if YoutubeIE.suitable(url) else super(
+        return not YoutubeIE.suitable(url) and super(
             YoutubeTabIE, cls).suitable(url)
 
-    def _extract_channel_id(self, webpage):
-        channel_id = self._html_search_meta(
-            'channelId', webpage, 'channel id', default=None)
-        if channel_id:
-            return channel_id
-        channel_url = self._html_search_meta(
-            ('og:url', 'al:ios:url', 'al:android:url', 'al:web:url',
-             'twitter:url', 'twitter:app:url:iphone', 'twitter:app:url:ipad',
-             'twitter:app:url:googleplay'), webpage, 'channel url')
-        return self._search_regex(
-            r'https?://(?:www\.)?youtube\.com/channel/([^/?#&])+',
-            channel_url, 'channel id')
-
     @staticmethod
     def _extract_grid_item_renderer(item):
         assert isinstance(item, dict)
@@ -3116,27 +3190,18 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
         else:
             raise ExtractorError('Unable to find selected tab')
 
-    @staticmethod
-    def _extract_uploader(data):
+    def _extract_uploader(self, metadata, data):
         uploader = {}
-        sidebar_renderer = try_get(
-            data, lambda x: x['sidebar']['playlistSidebarRenderer']['items'], list)
-        if sidebar_renderer:
-            for item in sidebar_renderer:
-                if not isinstance(item, dict):
-                    continue
-                renderer = item.get('playlistSidebarSecondaryInfoRenderer')
-                if not isinstance(renderer, dict):
-                    continue
-                owner = try_get(
-                    renderer, lambda x: x['videoOwner']['videoOwnerRenderer']['title']['runs'][0], dict)
-                if owner:
-                    uploader['uploader'] = owner.get('text')
-                    uploader['uploader_id'] = try_get(
-                        owner, lambda x: x['navigationEndpoint']['browseEndpoint']['browseId'], compat_str)
-                    uploader['uploader_url'] = urljoin(
-                        'https://www.youtube.com/',
-                        try_get(owner, lambda x: x['navigationEndpoint']['browseEndpoint']['canonicalBaseUrl'], compat_str))
+        renderers = traverse_obj(data,
+                                 ('sidebar', 'playlistSidebarRenderer', 'items'))
+        uploader['channel_id'] = self._extract_channel_id('', metadata=metadata, renderers=renderers)
+        uploader['uploader'] = (
+            self._extract_author_var('', 'name', renderers=renderers)
+            or self._extract_author_var('', 'name', metadata=metadata))
+        uploader['uploader_url'] = self._yt_urljoin(
+            self._extract_author_var('', 'url', metadata=metadata, renderers=renderers))
+        uploader['uploader_id'] = self._extract_uploader_id(uploader['uploader_url'])
+        uploader['channel'] = uploader['uploader']
         return uploader
 
     @staticmethod
@@ -3187,8 +3252,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             self._entries(selected_tab, item_id, webpage),
             playlist_id=playlist_id, playlist_title=title,
             playlist_description=description)
-        playlist.update(self._extract_uploader(data))
-        return playlist
+        return merge_dicts(playlist, self._extract_uploader(renderer, data))
 
     def _extract_from_playlist(self, item_id, url, data, playlist):
         title = playlist.get('title') or try_get(
@@ -3275,8 +3339,9 @@ class YoutubePlaylistIE(InfoExtractor):
         'info_dict': {
             'title': '[OLD]Team Fortress 2 (Class-based LP)',
             'id': 'PLBB231211A4F62143',
-            'uploader': 'Wickydoo',
-            'uploader_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
+            'uploader': 'Wickman',
+            'uploader_id': '@WickmanVT',
+            'channel_id': 'UCKSpbfbl5kRQpTdL7kMc-1Q',
         },
         'playlist_mincount': 29,
     }, {
@@ -3290,21 +3355,25 @@ class YoutubePlaylistIE(InfoExtractor):
     }, {
         'note': 'embedded',
         'url': 'https://www.youtube.com/embed/videoseries?list=PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
-        'playlist_count': 4,
+        # TODO: full playlist requires _reload_with_unavailable_videos()
+        # 'playlist_count': 4,
+        'playlist_mincount': 1,
         'info_dict': {
             'title': 'JODA15',
             'id': 'PL6IaIsEjSbf96XFRuNccS_RuEXwNdsoEu',
             'uploader': 'milan',
-            'uploader_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
+            'uploader_id': '@milan5503',
+            'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
         }
     }, {
         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
-        'playlist_mincount': 982,
+        'playlist_mincount': 455,
         'info_dict': {
             'title': '2018 Chinese New Singles (11/6 updated)',
             'id': 'PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
             'uploader': 'LBK',
-            'uploader_id': 'UC21nz3_MesPLqtDqwdvnoxA',
+            'uploader_id': '@music_king',
+            'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
         }
     }, {
         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
@@ -3342,8 +3411,8 @@ class YoutubeYtBeIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Small Scale Baler and Braiding Rugs',
             'uploader': 'Backus-Page House Museum',
-            'uploader_id': 'backuspagemuseum',
-            'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/backuspagemuseum',
+            'uploader_id': '@backuspagemuseum',
+            'uploader_url': r're:https?://(?:www\.)?youtube\.com/@backuspagemuseum',
             'upload_date': '20161008',
             'description': 'md5:800c0c78d5eb128500bffd4f0b4f2e8a',
             'categories': ['Nonprofits & Activism'],

From 3da17834a49fad2a97c308fdd89aa26781ef4d60 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Tue, 28 Feb 2023 23:03:44 +0530
Subject: [PATCH 611/860] [Youtube] Construct dash formats with `range` query

See yt-dlp/yt_dlp#6369
---
 youtube_dl/extractor/youtube.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4246d84f9..89711c84e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1694,8 +1694,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if n_response is None:
                 # give up if descrambling failed
                 break
-            fmt['url'] = update_url(
-                parsed_fmt_url, query_update={'n': [n_response]})
+            for fmt_dct in traverse_obj(fmt, (None, (None, ('fragments', Ellipsis))), expected_type=dict):
+                fmt_dct['url'] = update_url(
+                    fmt_dct['url'], query_update={'n': [n_response]})
 
     # from yt-dlp, with tweaks
     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
@@ -2047,10 +2048,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if no_video:
                 dct['abr'] = tbr
             if no_audio or no_video:
-                dct['downloader_options'] = {
-                    # Youtube throttles chunks >~10M
-                    'http_chunk_size': 10485760,
-                }
+                CHUNK_SIZE = 10 << 20
+                # avoid Youtube throttling
+                dct.update({
+                    'protocol': 'http_dash_segments',
+                    'fragments': [{
+                        'url': update_url_query(dct['url'], {
+                            'range': '{0}-{1}'.format(range_start, min(range_start + CHUNK_SIZE - 1, dct['filesize']))
+                        })
+                    } for range_start in range(0, dct['filesize'], CHUNK_SIZE)]
+                } if dct['filesize'] else {
+                    'downloader_options': {'http_chunk_size': CHUNK_SIZE}  # No longer useful?
+                })
+
                 if dct.get('ext'):
                     dct['container'] = dct['ext'] + '_dash'
             formats.append(dct)

From 3e92c60fcd94c37428d57153dbdd14cd0a1f9226 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Fri, 3 Mar 2023 16:48:54 +0530
Subject: [PATCH 612/860] [jsinterp] Handle `Date` at epoch 0

See yt-dlp/yt_dlp#6400
---
 test/test_youtube_signature.py | 4 ++++
 youtube_dl/jsinterp.py         | 2 +-
 2 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index ac37ffa45..decf7ee38 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -67,6 +67,10 @@ _SIG_TESTS = [
 ]
 
 _NSIG_TESTS = [
+    (
+        'https://www.youtube.com/s/player/7862ca1f/player_ias.vflset/en_US/base.js',
+        'X_LCxVDjAavgE5t', 'yxJ1dM6iz5ogUg',
+    ),
     (
         'https://www.youtube.com/s/player/9216d1f7/player_ias.vflset/en_US/base.js',
         'SLp9F5bwjAdhE9F-', 'gWnb9IK2DJ8Q1w',
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index a3bc42a61..e28670a3f 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -405,7 +405,7 @@ class JSInterpreter(object):
                 left, right = self._separate_at_paren(obj[len(klass):])
                 argvals = self.interpret_iter(left, local_vars, allow_recursion)
                 expr = konstr(*argvals)
-                if not expr:
+                if expr is None:
                     raise self.Exception('Failed to parse {klass} {left!r:.100}'.format(**locals()), expr=expr)
                 expr = self._dump(expr, local_vars) + right
                 break

From 040271022709c4d20d33c604d1dbc72dc2da472d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 5 Mar 2023 23:07:07 +0000
Subject: [PATCH 613/860] [jsinterp] Fix regexp parsing and .replace[All]
 method

 * For performance, make regexp object instantiation lazy
 * Other small performance improvements
---
 test/test_jsinterp.py  | 46 ++++++++++++++++++-----
 youtube_dl/jsinterp.py | 84 ++++++++++++++++++++++++++++--------------
 2 files changed, 93 insertions(+), 37 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index b5962356c..5d129433d 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -139,21 +139,16 @@ class TestJSInterpreter(unittest.TestCase):
         self.assertTrue(math.isnan(jsi.call_function('x')))
 
     def test_Date(self):
-        jsi = JSInterpreter('''
-        function x() { return new Date('Wednesday 31 December 1969 18:01:26 MDT') - 0; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 86000)
-
         jsi = JSInterpreter('''
         function x(dt) { return new Date(dt) - 0; }
         ''')
         self.assertEqual(jsi.call_function('x', 'Wednesday 31 December 1969 18:01:26 MDT'), 86000)
 
         # date format m/d/y
-        jsi = JSInterpreter('''
-        function x() { return new Date('12/31/1969 18:01:26 MDT') - 0; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 86000)
+        self.assertEqual(jsi.call_function('x', '12/31/1969 18:01:26 MDT'), 86000)
+
+        # epoch 0
+        self.assertEqual(jsi.call_function('x', '1 January 1970 00:00:00 UTC'), 0)
 
     def test_call(self):
         jsi = JSInterpreter('''
@@ -445,7 +440,7 @@ class TestJSInterpreter(unittest.TestCase):
         self.assertIs(jsi.call_function('x'), None)
 
         jsi = JSInterpreter('''
-        function x() { let a=/,,[/,913,/](,)}/; return a; }
+        function x() { let a=/,,[/,913,/](,)}/; "".replace(a, ""); return a; }
         ''')
         attrs = set(('findall', 'finditer', 'flags', 'groupindex',
                      'groups', 'match', 'pattern', 'scanner',
@@ -457,6 +452,31 @@ class TestJSInterpreter(unittest.TestCase):
         ''')
         self.assertEqual(jsi.call_function('x').flags & ~re.U, re.I)
 
+        jsi = JSInterpreter(r'''
+        function x() { let a="data-name".replace("data-", ""); return a }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 'name')
+
+        jsi = JSInterpreter(r'''
+        function x() { let a="data-name".replace(new RegExp("^.+-"), ""); return a; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 'name')
+
+        jsi = JSInterpreter(r'''
+        function x() { let a="data-name".replace(/^.+-/, ""); return a; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 'name')
+
+        jsi = JSInterpreter(r'''
+        function x() { let a="data-name".replace(/a/g, "o"); return a; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 'doto-nome')
+
+        jsi = JSInterpreter(r'''
+        function x() { let a="data-name".replaceAll("a", "o"); return a; }
+        ''')
+        self.assertEqual(jsi.call_function('x'), 'doto-nome')
+
         jsi = JSInterpreter(r'''
         function x() { let a=[/[)\\]/]; return a[0]; }
         ''')
@@ -485,6 +505,12 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('function x(){return 1236566549 << 5}')
         self.assertEqual(jsi.call_function('x'), 915423904)
 
+    """ # fails so far
+    def test_packed(self):
+        jsi = JSInterpreter('''function x(p,a,c,k,e,d){while(c--)if(k[c])p=p.replace(new RegExp('\\b'+c.toString(a)+'\\b','g'),k[c]);return p}''')
+        self.assertEqual(jsi.call_function('x', '''h 7=g("1j");7.7h({7g:[{33:"w://7f-7e-7d-7c.v.7b/7a/79/78/77/76.74?t=73&s=2s&e=72&f=2t&71=70.0.0.1&6z=6y&6x=6w"}],6v:"w://32.v.u/6u.31",16:"r%",15:"r%",6t:"6s",6r:"",6q:"l",6p:"l",6o:"6n",6m:\'6l\',6k:"6j",9:[{33:"/2u?b=6i&n=50&6h=w://32.v.u/6g.31",6f:"6e"}],1y:{6d:1,6c:\'#6b\',6a:\'#69\',68:"67",66:30,65:r,},"64":{63:"%62 2m%m%61%5z%5y%5x.u%5w%5v%5u.2y%22 2k%m%1o%22 5t%m%1o%22 5s%m%1o%22 2j%m%5r%22 16%m%5q%22 15%m%5p%22 5o%2z%5n%5m%2z",5l:"w://v.u/d/1k/5k.2y",5j:[]},\'5i\':{"5h":"5g"},5f:"5e",5d:"w://v.u",5c:{},5b:l,1x:[0.25,0.50,0.75,1,1.25,1.5,2]});h 1m,1n,5a;h 59=0,58=0;h 7=g("1j");h 2x=0,57=0,56=0;$.55({54:{\'53-52\':\'2i-51\'}});7.j(\'4z\',6(x){c(5>0&&x.1l>=5&&1n!=1){1n=1;$(\'q.4y\').4x(\'4w\')}});7.j(\'13\',6(x){2x=x.1l});7.j(\'2g\',6(x){2w(x)});7.j(\'4v\',6(){$(\'q.2v\').4u()});6 2w(x){$(\'q.2v\').4t();c(1m)19;1m=1;17=0;c(4s.4r===l){17=1}$.4q(\'/2u?b=4p&2l=1k&4o=2t-4n-4m-2s-4l&4k=&4j=&4i=&17=\'+17,6(2r){$(\'#4h\').4g(2r)});$(\'.3-8-4f-4e:4d("4c")\').2h(6(e){2q();g().4b(0);g().4a(l)});6 2q(){h $14=$("<q />").2p({1l:"49",16:"r%",15:"r%",48:0,2n:0,2o:47,46:"45(10%, 10%, 10%, 0.4)","44-43":"42"});$("<41 />").2p({16:"60%",15:"60%",2o:40,"3z-2n":"3y"}).3x({\'2m\':\'/?b=3w&2l=1k\',\'2k\':\'0\',\'2j\':\'2i\'}).2f($14);$14.2h(6(){$(3v).3u();g().2g()});$14.2f($(\'#1j\'))}g().13(0);}6 3t(){h 9=7.1b(2e);2d.2c(9);c(9.n>1){1r(i=0;i<9.n;i++){c(9[i].1a==2e){2d.2c(\'!!=\'+i);7.1p(i)}}}}7.j(\'3s\',6(){g().1h("/2a/3r.29","3q 10 28",6(){g().13(g().27()+10)},"2b");$("q[26=2b]").23().21(\'.3-20-1z\');g().1h("/2a/3p.29","3o 10 28",6(){h 12=g().27()-10;c(12<0)12=0;g().13(12)},"24");$("q[26=24]").23().21(\'.3-20-1z\');});6 1i(){}7.j(\'3n\',6(){1i()});7.j(\'3m\',6(){1i()});7.j("k",6(y){h 9=7.1b();c(9.n<2)19;$(\'.3-8-3l-3k\').3j(6(){$(\'#3-8-a-k\').1e(\'3-8-a-z\');$(\'.3-a-k\').p(\'o-1f\',\'11\')});7.1h("/3i/3h.3g","3f 3e",6(){$(\'.3-1w\').3d(\'3-8-1v\');$(\'.3-8-1y, .3-8-1x\').p(\'o-1g\',\'11\');c($(\'.3-1w\').3c(\'3-8-1v\')){$(\'.3-a-k\').p(\'o-1g\',\'l\');$(\'.3-a-k\').p(\'o-1f\',\'l\');$(\'.3-8-a\').1e(\'3-8-a-z\');$(\'.3-8-a:1u\').3b(\'3-8-a-z\')}3a{$(\'.3-a-k\').p(\'o-1g\',\'11\');$(\'.3-a-k\').p(\'o-1f\',\'11\');$(\'.3-8-a:1u\').1e(\'3-8-a-z\')}},"39");7.j("38",6(y){1d.37(\'1c\',y.9[y.36].1a)});c(1d.1t(\'1c\')){35("1s(1d.1t(\'1c\'));",34)}});h 18;6 1s(1q){h 9=7.1b();c(9.n>1){1r(i=0;i<9.n;i++){c(9[i].1a==1q){c(i==18){19}18=i;7.1p(i)}}}}',36,270,'|||jw|||function|player|settings|tracks|submenu||if||||jwplayer|var||on|audioTracks|true|3D|length|aria|attr|div|100|||sx|filemoon|https||event|active||false|tt|seek|dd|height|width|adb|current_audio|return|name|getAudioTracks|default_audio|localStorage|removeClass|expanded|checked|addButton|callMeMaybe|vplayer|0fxcyc2ajhp1|position|vvplay|vvad|220|setCurrentAudioTrack|audio_name|for|audio_set|getItem|last|open|controls|playbackRates|captions|rewind|icon|insertAfter||detach|ff00||button|getPosition|sec|png|player8|ff11|log|console|track_name|appendTo|play|click|no|scrolling|frameborder|file_code|src|top|zIndex|css|showCCform|data|1662367683|383371|dl|video_ad|doPlay|prevt|mp4|3E||jpg|thumbs|file|300|setTimeout|currentTrack|setItem|audioTrackChanged|dualSound|else|addClass|hasClass|toggleClass|Track|Audio|svg|dualy|images|mousedown|buttons|topbar|playAttemptFailed|beforePlay|Rewind|fr|Forward|ff|ready|set_audio_track|remove|this|upload_srt|prop|50px|margin|1000001|iframe|center|align|text|rgba|background|1000000|left|absolute|pause|setCurrentCaptions|Upload|contains|item|content|html|fviews|referer|prem|embed|3e57249ef633e0d03bf76ceb8d8a4b65|216|83|hash|view|get|TokenZir|window|hide|show|complete|slow|fadeIn|video_ad_fadein|time||cache|Cache|Content|headers|ajaxSetup|v2done|tott|vastdone2|vastdone1|vvbefore|playbackRateControls|cast|aboutlink|FileMoon|abouttext|UHD|1870|qualityLabels|sites|GNOME_POWER|link|2Fiframe|3C|allowfullscreen|22360|22640|22no|marginheight|marginwidth|2FGNOME_POWER|2F0fxcyc2ajhp1|2Fe|2Ffilemoon|2F|3A||22https|3Ciframe|code|sharing|fontOpacity|backgroundOpacity|Tahoma|fontFamily|303030|backgroundColor|FFFFFF|color|userFontScale|thumbnails|kind|0fxcyc2ajhp10000|url|get_slides|start|startparam|none|preload|html5|primary|hlshtml|androidhls|duration|uniform|stretching|0fxcyc2ajhp1_xt|image|2048|sp|6871|asn|127|srv|43200|_g3XlBcu2lmD9oDexD2NLWSmah2Nu3XcDrl93m9PwXY|m3u8||master|0fxcyc2ajhp1_x|00076|01|hls2|to|s01|delivery|storage|moon|sources|setup'''.split('|')))
+    """
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index e28670a3f..ab7d6f926 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -12,9 +12,11 @@ from .utils import (
     js_to_json,
     remove_quotes,
     unified_timestamp,
+    variadic,
 )
 from .compat import (
     compat_basestring,
+    compat_chr,
     compat_collections_chain_map as ChainMap,
     compat_itertools_zip_longest as zip_longest,
     compat_str,
@@ -205,10 +207,10 @@ class JSInterpreter(object):
             super(JSInterpreter.Exception, self).__init__(msg, *args, **kwargs)
 
     class JS_RegExp(object):
-        _RE_FLAGS = {
+        RE_FLAGS = {
             # special knowledge: Python's re flags are bitmask values, current max 128
             # invent new bitmask values well above that for literal parsing
-            # TODO: new pattern class to execute matches with these flags
+            # TODO: execute matches with these flags (remaining: d, y)
             'd': 1024,  # Generate indices for substring matches
             'g': 2048,  # Global search
             'i': re.I,  # Case-insensitive search
@@ -218,12 +220,19 @@ class JSInterpreter(object):
             'y': 4096,  # Perform a "sticky" search that matches starting at the current position in the target string
         }
 
-        def __init__(self, pattern_txt, flags=''):
+        def __init__(self, pattern_txt, flags=0):
             if isinstance(flags, compat_str):
                 flags, _ = self.regex_flags(flags)
-            # Thx: https://stackoverflow.com/questions/44773522/setattr-on-python2-sre-sre-pattern
             # First, avoid https://github.com/python/cpython/issues/74534
-            self.__self = re.compile(pattern_txt.replace('[[', r'[\['), flags)
+            self.__self = None
+            self.__pattern_txt = pattern_txt.replace('[[', r'[\[')
+            self.__flags = flags
+
+        def __instantiate(self):
+            if self.__self:
+                return
+            self.__self = re.compile(self.__pattern_txt, self.__flags)
+            # Thx: https://stackoverflow.com/questions/44773522/setattr-on-python2-sre-sre-pattern
             for name in dir(self.__self):
                 # Only these? Obviously __class__, __init__.
                 # PyPy creates a __weakref__ attribute with value None
@@ -232,15 +241,21 @@ class JSInterpreter(object):
                     continue
                 setattr(self, name, getattr(self.__self, name))
 
+        def __getattr__(self, name):
+            self.__instantiate()
+            if hasattr(self, name):
+                return getattr(self, name)
+            return super(JSInterpreter.JS_RegExp, self).__getattr__(name)
+
         @classmethod
         def regex_flags(cls, expr):
             flags = 0
             if not expr:
                 return flags, expr
             for idx, ch in enumerate(expr):
-                if ch not in cls._RE_FLAGS:
+                if ch not in cls.RE_FLAGS:
                     break
-                flags |= cls._RE_FLAGS[ch]
+                flags |= cls.RE_FLAGS[ch]
             return flags, expr[idx + 1:]
 
     @classmethod
@@ -265,17 +280,17 @@ class JSInterpreter(object):
         counters = dict((k, 0) for k in _MATCHING_PARENS.values())
         start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
         in_quote, escaping, skipping = None, False, 0
-        after_op, in_regex_char_group, skip_re = True, False, 0
+        after_op, in_regex_char_group = True, False
 
         for idx, char in enumerate(expr):
-            if skip_re > 0:
-                skip_re -= 1
-                continue
+            paren_delta = 0
             if not in_quote:
                 if char in _MATCHING_PARENS:
                     counters[_MATCHING_PARENS[char]] += 1
+                    paren_delta = 1
                 elif char in counters:
                     counters[char] -= 1
+                    paren_delta = -1
             if not escaping:
                 if char in _QUOTES and in_quote in (char, None):
                     if in_quote or after_op or char != '/':
@@ -283,7 +298,7 @@ class JSInterpreter(object):
                 elif in_quote == '/' and char in '[]':
                     in_regex_char_group = char == '['
             escaping = not escaping and in_quote and char == '\\'
-            after_op = not in_quote and (char in cls.OP_CHARS or (char.isspace() and after_op))
+            after_op = not in_quote and (char in cls.OP_CHARS or paren_delta > 0 or (after_op and char.isspace()))
 
             if char != delim[pos] or any(counters.values()) or in_quote:
                 pos = skipping = 0
@@ -293,7 +308,7 @@ class JSInterpreter(object):
                 continue
             elif pos == 0 and skip_delims:
                 here = expr[idx:]
-                for s in skip_delims if isinstance(skip_delims, (list, tuple)) else [skip_delims]:
+                for s in variadic(skip_delims):
                     if here.startswith(s) and s:
                         skipping = len(s) - 1
                         break
@@ -316,7 +331,7 @@ class JSInterpreter(object):
         separated = list(cls._separate(expr, delim, 1))
 
         if len(separated) < 2:
-            raise cls.Exception('No terminating paren {delim} in {expr}'.format(**locals()))
+            raise cls.Exception('No terminating paren {delim} in {expr!r:.5500}'.format(**locals()))
         return separated[0][1:].strip(), separated[1].strip()
 
     @staticmethod
@@ -361,6 +376,20 @@ class JSInterpreter(object):
         except TypeError:
             return self._named_object(namespace, obj)
 
+    # used below
+    _VAR_RET_THROW_RE = re.compile(r'''(?x)
+        (?P<var>(?:var|const|let)\s)|return(?:\s+|(?=["'])|$)|(?P<throw>throw\s+)
+        ''')
+    _COMPOUND_RE = re.compile(r'''(?x)
+        (?P<try>try)\s*\{|
+        (?P<if>if)\s*\(|
+        (?P<switch>switch)\s*\(|
+        (?P<for>for)\s*\(|
+        (?P<while>while)\s*\(
+        ''')
+    _FINALLY_RE = re.compile(r'finally\s*\{')
+    _SWITCH_RE = re.compile(r'switch\s*\(')
+
     def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if allow_recursion < 0:
             raise self.Exception('Recursion limit reached')
@@ -375,7 +404,7 @@ class JSInterpreter(object):
             if should_return:
                 return ret, should_return
 
-        m = re.match(r'(?P<var>(?:var|const|let)\s)|return(?:\s+|(?=["\'])|$)|(?P<throw>throw\s+)', stmt)
+        m = self._VAR_RET_THROW_RE.match(stmt)
         if m:
             expr = stmt[len(m.group(0)):].strip()
             if m.group('throw'):
@@ -447,13 +476,7 @@ class JSInterpreter(object):
                 for item in self._separate(inner)])
             expr = name + outer
 
-        m = re.match(r'''(?x)
-                (?P<try>try)\s*\{|
-                (?P<if>if)\s*\(|
-                (?P<switch>switch)\s*\(|
-                (?P<for>for)\s*\(|
-                (?P<while>while)\s*\(
-                ''', expr)
+        m = self._COMPOUND_RE.match(expr)
         md = m.groupdict() if m else {}
         if md.get('if'):
             cndn, expr = self._separate_at_paren(expr[m.end() - 1:])
@@ -512,7 +535,7 @@ class JSInterpreter(object):
                     err = None
                     pending = self.interpret_statement(sub_expr, catch_vars, allow_recursion)
 
-            m = re.match(r'finally\s*\{', expr)
+            m = self._FINALLY_RE.match(expr)
             if m:
                 sub_expr, expr = self._separate_at_paren(expr[m.end() - 1:])
                 ret, should_abort = self.interpret_statement(sub_expr, local_vars, allow_recursion)
@@ -531,7 +554,7 @@ class JSInterpreter(object):
             if remaining.startswith('{'):
                 body, expr = self._separate_at_paren(remaining)
             else:
-                switch_m = re.match(r'switch\s*\(', remaining)  # FIXME
+                switch_m = self._SWITCH_RE.match(remaining)  # FIXME
                 if switch_m:
                     switch_val, remaining = self._separate_at_paren(remaining[switch_m.end() - 1:])
                     body, expr = self._separate_at_paren(remaining, '}')
@@ -735,7 +758,7 @@ class JSInterpreter(object):
                 if obj == compat_str:
                     if member == 'fromCharCode':
                         assertion(argvals, 'takes one or more arguments')
-                        return ''.join(map(chr, argvals))
+                        return ''.join(map(compat_chr, argvals))
                     raise self.Exception('Unsupported string method ' + member, expr=expr)
                 elif obj == float:
                     if member == 'pow':
@@ -808,10 +831,17 @@ class JSInterpreter(object):
                     if idx >= len(obj):
                         return None
                     return ord(obj[idx])
-                elif member == 'replace':
+                elif member in ('replace', 'replaceAll'):
                     assertion(isinstance(obj, compat_str), 'must be applied on a string')
                     assertion(len(argvals) == 2, 'takes exactly two arguments')
-                    return re.sub(argvals[0], argvals[1], obj)
+                    # TODO: argvals[1] callable, other Py vs JS edge cases
+                    if isinstance(argvals[0], self.JS_RegExp):
+                        count = 0 if argvals[0].flags & self.JS_RegExp.RE_FLAGS['g'] else 1
+                        assertion(member != 'replaceAll' or count == 0,
+                                  'replaceAll must be called with a global RegExp')
+                        return argvals[0].sub(argvals[1], obj, count=count)
+                    count = ('replaceAll', 'replace').index(member)
+                    return re.sub(re.escape(argvals[0]), argvals[1], obj, count=count)
 
                 idx = int(member) if isinstance(obj, list) else member
                 return obj[idx](argvals, allow_recursion=allow_recursion)

From 27d41d73655b8fbf2dedf88cac96220520d526b5 Mon Sep 17 00:00:00 2001
From: Sophira <github@theblob.org>
Date: Tue, 7 Mar 2023 15:49:31 +0000
Subject: [PATCH 614/860] [doc] Recommend "Get cookies.txt LOCALLY" extension
 in README.md (#31763)

* remove link to suspect "Get cookies.txt" extension, dropped from Chrome store
* link to new Manifest V3-compatible open-source "Get cookies.txt LOCALLY" extension.

Fixes #31465.
---
 README.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 6e07ddb1c..227e34046 100644
--- a/README.md
+++ b/README.md
@@ -918,7 +918,7 @@ Either prepend `https://www.youtube.com/watch?v=` or separate the ID from the op
 
 Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`.
 
-In order to extract cookies from browser use any conforming browser extension for exporting cookies. For example, [Get cookies.txt](https://chrome.google.com/webstore/detail/get-cookiestxt/bgaddhkoddajcdgocldbbfleckgcbcid/) (for Chrome) or [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) (for Firefox).
+In order to extract cookies from browser use any conforming browser extension for exporting cookies. For example, [Get cookies.txt LOCALLY](https://chrome.google.com/webstore/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc) (for Chrome) or [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) (for Firefox).
 
 Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows and `LF` (`\n`) for Unix and Unix-like systems (Linux, macOS, etc.). `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format.
 

From 8c86fd33dca48ebb505ed04150d9e35993b9fe7e Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 9 Mar 2023 16:40:30 +0000
Subject: [PATCH 615/860] [doc] Improve "guidance" on bug reporting

---
 README.md | 22 +++++++++++++---------
 1 file changed, 13 insertions(+), 9 deletions(-)

diff --git a/README.md b/README.md
index 227e34046..14a3d6c86 100644
--- a/README.md
+++ b/README.md
@@ -1408,7 +1408,11 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl:
 
 # BUGS
 
-Bugs and suggestions should be reported at: <https://github.com/ytdl-org/youtube-dl/issues>. Unless you were prompted to or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the IRC channel [#youtube-dl](irc://chat.freenode.net/#youtube-dl) on freenode ([webchat](https://webchat.freenode.net/?randomnick=1&channels=youtube-dl)).
+Bugs and suggestions should be reported in the issue tracker: <https://github.com/ytdl-org/youtube-dl/issues> (<https://yt-dl.org/bug> is an alias for this). Unless you were prompted to or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the IRC channel [#youtube-dl](irc://chat.freenode.net/#youtube-dl) on freenode ([webchat](https://webchat.freenode.net/?randomnick=1&channels=youtube-dl)).
+
+## Opening a bug report or suggestion
+
+Be sure to follow instructions provided **below** and **in the issue tracker**. Complete the appropriate issue template fully. Consider whether your problem is covered by an existing issue: if so, follow the discussion there. Avoid commenting on existing duplicate issues as such comments do not add to the discussion of the issue and are liable to be treated as spam.
 
 **Please include the full output of youtube-dl when run with `-v`**, i.e. **add** `-v` flag to **your command line**, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this:
 ```
@@ -1428,17 +1432,17 @@ $ youtube-dl -v <your command line>
 
 The output (including the first lines) contains important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever.
 
-Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist):
+Finally please review your issue to avoid various common mistakes (you can and should use this as a checklist) listed below.
 
 ### Is the description of the issue itself sufficient?
 
-We often get issue reports that we cannot really decipher. While in most cases we eventually get the required information after asking back multiple times, this poses an unnecessary drain on our resources. Many contributors, including myself, are also not native speakers, so we may misread some parts.
+We often get issue reports that are hard to understand. To avoid subsequent clarifications, and to assist participants who are not native English speakers, please elaborate on what feature you are requesting, or what bug you want to be fixed.
 
-So please elaborate on what feature you are requesting, or what bug you want to be fixed. Make sure that it's obvious
+Make sure that it's obvious
 
 - What the problem is
 - How it could be fixed
-- How your proposed solution would look like
+- How your proposed solution would look
 
 If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a committer myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over.
 
@@ -1448,14 +1452,14 @@ If your server has multiple IPs or you suspect censorship, adding `--call-home`
 
 **Site support requests must contain an example URL**. An example URL is a URL you might want to download, like `https://www.youtube.com/watch?v=BaW_jenozKc`. There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. `https://www.youtube.com/`) is *not* an example URL.
 
+###  Is the issue already documented?
+
+Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or browse the [GitHub Issues](https://github.com/ytdl-org/youtube-dl/search?type=Issues) of this repository. Initially, at least, use the search term `-label:duplicate` to focus on active issues. If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity.
+
 ###  Are you using the latest version?
 
 Before reporting any issue, type `youtube-dl -U`. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well.
 
-###  Is the issue already documented?
-
-Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or browse the [GitHub Issues](https://github.com/ytdl-org/youtube-dl/search?type=Issues) of this repository. If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity.
-
 ###  Why are existing options not enough?
 
 Before requesting a new feature, please have a quick peek at [the list of supported options](https://github.com/ytdl-org/youtube-dl/blob/master/README.md#options). Many feature requests are for features that actually exist already! Please, absolutely do show off your work in the issue report and detail how the existing similar options do *not* solve your problem.

From 5c985d4f81a43ada75dafb23233e7fe39913907a Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 11 Mar 2023 12:09:55 +0000
Subject: [PATCH 616/860] [downloader] Let _ffmpeg_ handle DASH segments

Fixes https://github.com/ytdl-org/youtube-dl/issues/31792 after 3da1783.
---
 youtube_dl/downloader/external.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index bffcd10b6..1b6bd1fa2 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -273,7 +273,7 @@ class HttpieFD(ExternalFD):
 class FFmpegFD(ExternalFD):
     @classmethod
     def supports(cls, info_dict):
-        return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms')
+        return info_dict['protocol'] in ('http', 'https', 'ftp', 'ftps', 'm3u8', 'rtsp', 'rtmp', 'mms', 'http_dash_segments')
 
     @classmethod
     def available(cls):

From baa6c5e95cb307e7d716645780ff8aef22de6aca Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 11 Mar 2023 12:17:00 +0000
Subject: [PATCH 617/860] [FragmentFD] Respect `--no-continue`

* discard partial fragment on `--no-continue`
* continue with correct progress display otherwise

Resolves #21467
---
 youtube_dl/downloader/common.py   | 24 +++++++++++-----
 youtube_dl/downloader/dash.py     | 10 +++----
 youtube_dl/downloader/fragment.py | 46 +++++++++++++++++++++----------
 youtube_dl/downloader/http.py     | 15 ++++------
 4 files changed, 58 insertions(+), 37 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 1cdba89cd..c86ce2aa5 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -88,17 +88,21 @@ class FileDownloader(object):
             return '---.-%'
         return '%6s' % ('%3.1f%%' % percent)
 
-    @staticmethod
-    def calc_eta(start, now, total, current):
+    @classmethod
+    def calc_eta(cls, start_or_rate, now_or_remaining, *args):
+        if len(args) < 2:
+            rate, remaining = (start_or_rate, now_or_remaining)
+            if None in (rate, remaining):
+                return None
+            return int(float(remaining) / rate)
+        start, now = (start_or_rate, now_or_remaining)
+        total, current = args
         if total is None:
             return None
         if now is None:
             now = time.time()
-        dif = now - start
-        if current == 0 or dif < 0.001:  # One millisecond
-            return None
-        rate = float(current) / dif
-        return int((float(total) - float(current)) / rate)
+        rate = cls.calc_speed(start, now, current)
+        return rate and int((float(total) - float(current)) / rate)
 
     @staticmethod
     def format_eta(eta):
@@ -123,6 +127,12 @@ class FileDownloader(object):
     def format_retries(retries):
         return 'inf' if retries == float('inf') else '%.0f' % retries
 
+    @staticmethod
+    def filesize_or_none(unencoded_filename):
+        fn = encodeFilename(unencoded_filename)
+        if os.path.isfile(fn):
+            return os.path.getsize(fn)
+
     @staticmethod
     def best_block_size(elapsed_time, bytes):
         new_min = max(bytes / 2.0, 1.0)
diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index c6d674bc6..cc30485f8 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -38,8 +38,7 @@ class DashSegmentsFD(FragmentFD):
             # In DASH, the first segment contains necessary headers to
             # generate a valid MP4 file, so always abort for the first segment
             fatal = i == 0 or not skip_unavailable_fragments
-            count = 0
-            while count <= fragment_retries:
+            for count in range(fragment_retries + 1):
                 try:
                     fragment_url = fragment.get('url')
                     if not fragment_url:
@@ -57,9 +56,8 @@ class DashSegmentsFD(FragmentFD):
                     # is usually enough) thus allowing to download the whole file successfully.
                     # To be future-proof we will retry all fragments that fail with any
                     # HTTP error.
-                    count += 1
-                    if count <= fragment_retries:
-                        self.report_retry_fragment(err, frag_index, count, fragment_retries)
+                    if count < fragment_retries:
+                        self.report_retry_fragment(err, frag_index, count + 1, fragment_retries)
                 except DownloadError:
                     # Don't retry fragment if error occurred during HTTP downloading
                     # itself since it has own retry settings
@@ -68,7 +66,7 @@ class DashSegmentsFD(FragmentFD):
                         break
                     raise
 
-            if count > fragment_retries:
+            if count >= fragment_retries:
                 if not fatal:
                     self.report_skip_fragment(frag_index)
                     continue
diff --git a/youtube_dl/downloader/fragment.py b/youtube_dl/downloader/fragment.py
index 35c76feba..913e91b64 100644
--- a/youtube_dl/downloader/fragment.py
+++ b/youtube_dl/downloader/fragment.py
@@ -71,7 +71,7 @@ class FragmentFD(FileDownloader):
 
     @staticmethod
     def __do_ytdl_file(ctx):
-        return not ctx['live'] and not ctx['tmpfilename'] == '-'
+        return ctx['live'] is not True and ctx['tmpfilename'] != '-'
 
     def _read_ytdl_file(self, ctx):
         assert 'ytdl_corrupt' not in ctx
@@ -101,6 +101,13 @@ class FragmentFD(FileDownloader):
             'url': frag_url,
             'http_headers': headers or info_dict.get('http_headers'),
         }
+        frag_resume_len = 0
+        if ctx['dl'].params.get('continuedl', True):
+            frag_resume_len = self.filesize_or_none(
+                self.temp_name(fragment_filename))
+        fragment_info_dict['frag_resume_len'] = frag_resume_len
+        ctx['frag_resume_len'] = frag_resume_len or 0
+
         success = ctx['dl'].download(fragment_filename, fragment_info_dict)
         if not success:
             return False, None
@@ -124,9 +131,7 @@ class FragmentFD(FileDownloader):
             del ctx['fragment_filename_sanitized']
 
     def _prepare_frag_download(self, ctx):
-        if 'live' not in ctx:
-            ctx['live'] = False
-        if not ctx['live']:
+        if not ctx.setdefault('live', False):
             total_frags_str = '%d' % ctx['total_frags']
             ad_frags = ctx.get('ad_frags', 0)
             if ad_frags:
@@ -136,10 +141,11 @@ class FragmentFD(FileDownloader):
         self.to_screen(
             '[%s] Total fragments: %s' % (self.FD_NAME, total_frags_str))
         self.report_destination(ctx['filename'])
+        continuedl = self.params.get('continuedl', True)
         dl = HttpQuietDownloader(
             self.ydl,
             {
-                'continuedl': True,
+                'continuedl': continuedl,
                 'quiet': True,
                 'noprogress': True,
                 'ratelimit': self.params.get('ratelimit'),
@@ -150,12 +156,11 @@ class FragmentFD(FileDownloader):
         )
         tmpfilename = self.temp_name(ctx['filename'])
         open_mode = 'wb'
-        resume_len = 0
 
         # Establish possible resume length
-        if os.path.isfile(encodeFilename(tmpfilename)):
+        resume_len = self.filesize_or_none(tmpfilename) or 0
+        if resume_len > 0:
             open_mode = 'ab'
-            resume_len = os.path.getsize(encodeFilename(tmpfilename))
 
         # Should be initialized before ytdl file check
         ctx.update({
@@ -164,7 +169,8 @@ class FragmentFD(FileDownloader):
         })
 
         if self.__do_ytdl_file(ctx):
-            if os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename']))):
+            ytdl_file_exists = os.path.isfile(encodeFilename(self.ytdl_filename(ctx['filename'])))
+            if continuedl and ytdl_file_exists:
                 self._read_ytdl_file(ctx)
                 is_corrupt = ctx.get('ytdl_corrupt') is True
                 is_inconsistent = ctx['fragment_index'] > 0 and resume_len == 0
@@ -178,7 +184,12 @@ class FragmentFD(FileDownloader):
                     if 'ytdl_corrupt' in ctx:
                         del ctx['ytdl_corrupt']
                     self._write_ytdl_file(ctx)
+
             else:
+                if not continuedl:
+                    if ytdl_file_exists:
+                        self._read_ytdl_file(ctx)
+                    ctx['fragment_index'] = resume_len = 0
                 self._write_ytdl_file(ctx)
                 assert ctx['fragment_index'] == 0
 
@@ -209,6 +220,7 @@ class FragmentFD(FileDownloader):
         start = time.time()
         ctx.update({
             'started': start,
+            'fragment_started': start,
             # Amount of fragment's bytes downloaded by the time of the previous
             # frag progress hook invocation
             'prev_frag_downloaded_bytes': 0,
@@ -218,6 +230,9 @@ class FragmentFD(FileDownloader):
             if s['status'] not in ('downloading', 'finished'):
                 return
 
+            if not total_frags and ctx.get('fragment_count'):
+                state['fragment_count'] = ctx['fragment_count']
+
             time_now = time.time()
             state['elapsed'] = time_now - start
             frag_total_bytes = s.get('total_bytes') or 0
@@ -232,16 +247,17 @@ class FragmentFD(FileDownloader):
                 ctx['fragment_index'] = state['fragment_index']
                 state['downloaded_bytes'] += frag_total_bytes - ctx['prev_frag_downloaded_bytes']
                 ctx['complete_frags_downloaded_bytes'] = state['downloaded_bytes']
+                ctx['speed'] = state['speed'] = self.calc_speed(
+                    ctx['fragment_started'], time_now, frag_total_bytes)
+                ctx['fragment_started'] = time.time()
                 ctx['prev_frag_downloaded_bytes'] = 0
             else:
                 frag_downloaded_bytes = s['downloaded_bytes']
                 state['downloaded_bytes'] += frag_downloaded_bytes - ctx['prev_frag_downloaded_bytes']
+                ctx['speed'] = state['speed'] = self.calc_speed(
+                    ctx['fragment_started'], time_now, frag_downloaded_bytes - ctx['frag_resume_len'])
                 if not ctx['live']:
-                    state['eta'] = self.calc_eta(
-                        start, time_now, estimated_size - resume_len,
-                        state['downloaded_bytes'] - resume_len)
-                state['speed'] = s.get('speed') or ctx.get('speed')
-                ctx['speed'] = state['speed']
+                    state['eta'] = self.calc_eta(state['speed'], estimated_size - state['downloaded_bytes'])
                 ctx['prev_frag_downloaded_bytes'] = frag_downloaded_bytes
             self._hook_progress(state)
 
@@ -268,7 +284,7 @@ class FragmentFD(FileDownloader):
                         os.utime(ctx['filename'], (time.time(), filetime))
                     except Exception:
                         pass
-            downloaded_bytes = os.path.getsize(encodeFilename(ctx['filename']))
+            downloaded_bytes = self.filesize_or_none(ctx['filename']) or 0
 
         self._hook_progress({
             'downloaded_bytes': downloaded_bytes,
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index d8ac41dcc..440471aa0 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -58,9 +58,9 @@ class HttpFD(FileDownloader):
 
         if self.params.get('continuedl', True):
             # Establish possible resume length
-            if os.path.isfile(encodeFilename(ctx.tmpfilename)):
-                ctx.resume_len = os.path.getsize(
-                    encodeFilename(ctx.tmpfilename))
+            ctx.resume_len = info_dict.get('frag_resume_len')
+            if ctx.resume_len is None:
+                ctx.resume_len = self.filesize_or_none(ctx.tmpfilename) or 0
 
         ctx.is_resume = ctx.resume_len > 0
 
@@ -115,9 +115,9 @@ class HttpFD(FileDownloader):
                         raise RetryDownload(err)
                     raise err
                 # When trying to resume, Content-Range HTTP header of response has to be checked
-                # to match the value of requested Range HTTP header. This is due to a webservers
+                # to match the value of requested Range HTTP header. This is due to webservers
                 # that don't support resuming and serve a whole file with no Content-Range
-                # set in response despite of requested Range (see
+                # set in response despite requested Range (see
                 # https://github.com/ytdl-org/youtube-dl/issues/6057#issuecomment-126129799)
                 if has_range:
                     content_range = ctx.data.headers.get('Content-Range')
@@ -293,10 +293,7 @@ class HttpFD(FileDownloader):
 
                 # Progress message
                 speed = self.calc_speed(start, now, byte_counter - ctx.resume_len)
-                if ctx.data_len is None:
-                    eta = None
-                else:
-                    eta = self.calc_eta(start, time.time(), ctx.data_len - ctx.resume_len, byte_counter - ctx.resume_len)
+                eta = self.calc_eta(speed, ctx.data_len and (ctx.data_len - ctx.resume_len))
 
                 self._hook_progress({
                     'status': 'downloading',

From e8de54bce50f6f77a4d7e8e80675f7003d5bf630 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 13 Mar 2023 19:45:54 +0000
Subject: [PATCH 618/860] [core] Handle `/../` sequences in HTTP URLs

* use Python's RFC implementation for embedded sequences
* hack: strip unbalanced leading `../` from path, like eg Firefox

See https://github.com/yt-dlp/yt-dlp/issues/3355
---
 youtube_dl/YoutubeDL.py | 24 ++++++++++++++++++++++++
 1 file changed, 24 insertions(+)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 8e8546596..bcf781744 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -39,6 +39,7 @@ from .compat import (
     compat_str,
     compat_tokenize_tokenize,
     compat_urllib_error,
+    compat_urllib_parse,
     compat_urllib_request,
     compat_urllib_request_DataHandler,
 )
@@ -60,6 +61,7 @@ from .utils import (
     format_bytes,
     formatSeconds,
     GeoRestrictedError,
+    HEADRequest,
     int_or_none,
     ISO3166Utils,
     locked_file,
@@ -74,6 +76,7 @@ from .utils import (
     preferredencoding,
     prepend_extension,
     process_communicate_or_kill,
+    PUTRequest,
     register_socks_protocols,
     render_table,
     replace_extension,
@@ -2297,6 +2300,27 @@ class YoutubeDL(object):
         """ Start an HTTP download """
         if isinstance(req, compat_basestring):
             req = sanitized_Request(req)
+        # an embedded /../ sequence is not automatically handled by urllib2
+        # see https://github.com/yt-dlp/yt-dlp/issues/3355
+        url = req.get_full_url()
+        parts = url.partition('/../')
+        if parts[1]:
+            url = compat_urllib_parse.urljoin(parts[0] + parts[1][:1], parts[1][1:] + parts[2])
+        if url:
+            # worse, URL path may have initial /../ against RFCs: work-around
+            # by stripping such prefixes, like eg Firefox
+            parts = compat_urllib_parse.urlsplit(url)
+            path = parts.path
+            while path.startswith('/../'):
+                path = path[3:]
+            url = parts._replace(path=path).geturl()
+            # get a new Request with the munged URL
+            if url != req.get_full_url():
+                req_type = {'HEAD': HEADRequest, 'PUT': PUTRequest}.get(
+                    req.get_method(), compat_urllib_request.Request)
+                req = req_type(
+                    url, data=req.data, headers=dict(req.header_items()),
+                    origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
         return self._opener.open(req, timeout=self._socket_timeout)
 
     def print_debug_header(self):

From 70ff01391068c98b4377c5cc17a8d00d5645e734 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 14 Mar 2023 00:58:59 +0000
Subject: [PATCH 619/860] [devscripts] Add a hack to convert command-line
 options to API options

---
 devscripts/cli_to_api.py | 64 ++++++++++++++++++++++++++++++++++++++++
 1 file changed, 64 insertions(+)
 create mode 100755 devscripts/cli_to_api.py

diff --git a/devscripts/cli_to_api.py b/devscripts/cli_to_api.py
new file mode 100755
index 000000000..2f4d6a458
--- /dev/null
+++ b/devscripts/cli_to_api.py
@@ -0,0 +1,64 @@
+#!/usr/bin/env python
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+"""
+This script displays the API parameters corresponding to a yt-dl command line
+
+Example:
+$ ./cli_to_api.py -f best
+{u'format': 'best'}
+$
+"""
+
+# Allow direct execution
+import os
+import sys
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+import youtube_dl
+from types import MethodType
+
+
+def cli_to_api(*opts):
+    YDL = youtube_dl.YoutubeDL
+
+    # to extract the parsed options, break out of YoutubeDL instantiation
+
+    # return options via this Exception
+    class ParseYTDLResult(Exception):
+        def __init__(self, result):
+            super(ParseYTDLResult, self).__init__('result')
+            self.opts = result
+
+    # replacement constructor that raises ParseYTDLResult
+    def ytdl_init(ydl, ydl_opts):
+        super(YDL, ydl).__init__(ydl_opts)
+        raise ParseYTDLResult(ydl_opts)
+
+    # patch in the constructor
+    YDL.__init__ = MethodType(ytdl_init, YDL)
+
+    # core parser
+    def parsed_options(argv):
+        try:
+            youtube_dl._real_main(list(argv))
+        except ParseYTDLResult as result:
+            return result.opts
+
+    # from https://github.com/yt-dlp/yt-dlp/issues/5859#issuecomment-1363938900
+    default = parsed_options([])
+    diff = dict((k, v) for k, v in parsed_options(opts).items() if default[k] != v)
+    if 'postprocessors' in diff:
+        diff['postprocessors'] = [pp for pp in diff['postprocessors'] if pp not in default['postprocessors']]
+    return diff
+
+
+def main():
+    from pprint import pprint
+    pprint(cli_to_api(*sys.argv))
+
+
+if __name__ == '__main__':
+    main()

From 6fece0a96b3cd8677f5c1185a57c6e21403fcb44 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 14 Mar 2023 13:01:32 +0000
Subject: [PATCH 620/860] [AENetworksBaseIE] Report missing show data instead
 of crash

---
 youtube_dl/extractor/aenetworks.py | 23 ++++++++++++++++++-----
 1 file changed, 18 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/aenetworks.py b/youtube_dl/extractor/aenetworks.py
index 2a1f08e39..59fbe048a 100644
--- a/youtube_dl/extractor/aenetworks.py
+++ b/youtube_dl/extractor/aenetworks.py
@@ -8,6 +8,8 @@ from ..utils import (
     ExtractorError,
     GeoRestrictedError,
     int_or_none,
+    remove_start,
+    traverse_obj,
     update_url_query,
     urlencode_postdata,
 )
@@ -33,14 +35,17 @@ class AENetworksBaseIE(ThePlatformIE):
     }
 
     def _extract_aen_smil(self, smil_url, video_id, auth=None):
-        query = {'mbr': 'true'}
+        query = {
+            'mbr': 'true',
+            'formats': 'M3U+none,MPEG-DASH+none,MPEG4,MP3',
+        }
         if auth:
             query['auth'] = auth
         TP_SMIL_QUERY = [{
             'assetTypes': 'high_video_ak',
-            'switch': 'hls_high_ak'
+            'switch': 'hls_high_ak',
         }, {
-            'assetTypes': 'high_video_s3'
+            'assetTypes': 'high_video_s3',
         }, {
             'assetTypes': 'high_video_s3',
             'switch': 'hls_high_fastly',
@@ -75,7 +80,14 @@ class AENetworksBaseIE(ThePlatformIE):
         requestor_id, brand = self._DOMAIN_MAP[domain]
         result = self._download_json(
             'https://feeds.video.aetnd.com/api/v2/%s/videos' % brand,
-            filter_value, query={'filter[%s]' % filter_key: filter_value})['results'][0]
+            filter_value, query={'filter[%s]' % filter_key: filter_value})
+        result = traverse_obj(
+            result, ('results',
+                     lambda k, v: k == 0 and v[filter_key] == filter_value),
+            get_all=False)
+        if not result:
+            raise ExtractorError('Show not found in A&E feed (too new?)', expected=True,
+                                 video_id=remove_start(filter_value, '/'))
         title = result['title']
         video_id = result['id']
         media_url = result['publicUrl']
@@ -126,7 +138,7 @@ class AENetworksIE(AENetworksBaseIE):
             'skip_download': True,
         },
         'add_ie': ['ThePlatform'],
-        'skip': 'This video is only available for users of participating TV providers.',
+        'skip': 'Geo-restricted - This content is not available in your location.'
     }, {
         'url': 'http://www.aetv.com/shows/duck-dynasty/season-9/episode-1',
         'info_dict': {
@@ -143,6 +155,7 @@ class AENetworksIE(AENetworksBaseIE):
             'skip_download': True,
         },
         'add_ie': ['ThePlatform'],
+        'skip': 'This video is only available for users of participating TV providers.',
     }, {
         'url': 'http://www.fyi.tv/shows/tiny-house-nation/season-1/episode-8',
         'only_matching': True

From 45495228b7a6728b7e764bbcf1f38490cd3d8697 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 19 Mar 2023 00:51:44 +0000
Subject: [PATCH 621/860] [downloader/http] Only check for resumability when
 actually resuming

---
 test/test_downloader_http.py  | 2 +-
 youtube_dl/downloader/http.py | 3 ++-
 2 files changed, 3 insertions(+), 2 deletions(-)

diff --git a/test/test_downloader_http.py b/test/test_downloader_http.py
index 4e6d7a2a0..6af86ae48 100644
--- a/test/test_downloader_http.py
+++ b/test/test_downloader_http.py
@@ -88,7 +88,7 @@ class TestHttpFD(unittest.TestCase):
         self.assertTrue(downloader.real_download(filename, {
             'url': 'http://127.0.0.1:%d/%s' % (self.port, ep),
         }))
-        self.assertEqual(os.path.getsize(encodeFilename(filename)), TEST_SIZE)
+        self.assertEqual(os.path.getsize(encodeFilename(filename)), TEST_SIZE, ep)
         try_rm(encodeFilename(filename))
 
     def download_all(self, params):
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 440471aa0..28a49b9e8 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -141,7 +141,8 @@ class HttpFD(FileDownloader):
                     # Content-Range is either not present or invalid. Assuming remote webserver is
                     # trying to send the whole file, resume is not possible, so wiping the local file
                     # and performing entire redownload
-                    self.report_unable_to_resume()
+                    if range_start > 0:
+                        self.report_unable_to_resume()
                     ctx.resume_len = 0
                     ctx.open_mode = 'wb'
                 ctx.data_len = int_or_none(ctx.data.info().get('Content-length', None))

From f35b757c826027ab5263d431bbe363c6403bd66d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 19 Mar 2023 02:27:46 +0000
Subject: [PATCH 622/860] [utils] Ensure `allow_types` for `variadic()` is a
 tuple

---
 test/test_utils.py  | 1 +
 youtube_dl/utils.py | 2 ++
 2 files changed, 3 insertions(+)

diff --git a/test/test_utils.py b/test/test_utils.py
index ea2b96ed2..b85d397d0 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1563,6 +1563,7 @@ Line 1
         self.assertEqual(variadic(None), (None, ))
         self.assertEqual(variadic('spam'), ('spam', ))
         self.assertEqual(variadic('spam', allowed_types=dict), 'spam')
+        self.assertEqual(variadic('spam', allowed_types=[dict]), 'spam')
 
     def test_traverse_obj(self):
         _TEST_DATA = {
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 761edcd49..f3c7af437 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -4213,6 +4213,8 @@ def multipart_encode(data, boundary=None):
 
 
 def variadic(x, allowed_types=(compat_str, bytes, dict)):
+    if not isinstance(allowed_types, tuple) and isinstance(allowed_types, compat_collections_abc.Iterable):
+        allowed_types = tuple(allowed_types)
     return x if isinstance(x, compat_collections_abc.Iterable) and not isinstance(x, allowed_types) else (x,)
 
 

From 88f28f620bcae7ba7302f8b049b74f0f8a12831f Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Sun, 12 Mar 2023 14:46:09 +0530
Subject: [PATCH 623/860] [extractor/youtube] Construct fragment list lazily

Ref: yt-dlp/yt-dlp/commit/e389d17
See: yt-dlp/yt-dlp#6517
---
 youtube_dl/extractor/youtube.py | 18 ++++++++++++------
 1 file changed, 12 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 89711c84e..6b153193c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -31,6 +31,7 @@ from ..utils import (
     get_element_by_attribute,
     int_or_none,
     js_to_json,
+    LazyList,
     merge_dicts,
     mimetype2ext,
     parse_codecs,
@@ -1986,9 +1987,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         itags = []
         itag_qualities = {}
         q = qualities(['tiny', 'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'])
+        CHUNK_SIZE = 10 << 20
+
         streaming_data = player_response.get('streamingData') or {}
         streaming_formats = streaming_data.get('formats') or []
         streaming_formats.extend(streaming_data.get('adaptiveFormats') or [])
+
+        def build_fragments(f):
+            return LazyList({
+                'url': update_url_query(f['url'], {
+                    'range': '{0}-{1}'.format(range_start, min(range_start + CHUNK_SIZE - 1, f['filesize']))
+                })
+            } for range_start in range(0, f['filesize'], CHUNK_SIZE))
+
         for fmt in streaming_formats:
             if fmt.get('targetDurationSec') or fmt.get('drmFamilies'):
                 continue
@@ -2048,15 +2059,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if no_video:
                 dct['abr'] = tbr
             if no_audio or no_video:
-                CHUNK_SIZE = 10 << 20
                 # avoid Youtube throttling
                 dct.update({
                     'protocol': 'http_dash_segments',
-                    'fragments': [{
-                        'url': update_url_query(dct['url'], {
-                            'range': '{0}-{1}'.format(range_start, min(range_start + CHUNK_SIZE - 1, dct['filesize']))
-                        })
-                    } for range_start in range(0, dct['filesize'], CHUNK_SIZE)]
+                    'fragments': build_fragments(dct),
                 } if dct['filesize'] else {
                     'downloader_options': {'http_chunk_size': CHUNK_SIZE}  # No longer useful?
                 })

From 3f6d2bd76f3393eef90896dfabc2d8dde37c2009 Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 9 Mar 2023 22:09:23 +0530
Subject: [PATCH 624/860] [extractor/youtube] Bypass throttling for `-f17`

and related cleanup

Thanks @AudricV for the finding

Ref: yt-dlp/yt-dlp/commit/c9abebb
---
 youtube_dl/extractor/youtube.py | 13 ++++---------
 1 file changed, 4 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6b153193c..ae3416b20 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2052,13 +2052,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 if mobj:
                     dct['ext'] = mimetype2ext(mobj.group(1))
                     dct.update(parse_codecs(mobj.group(2)))
-            no_audio = dct.get('acodec') == 'none'
-            no_video = dct.get('vcodec') == 'none'
-            if no_audio:
-                dct['vbr'] = tbr
-            if no_video:
-                dct['abr'] = tbr
-            if no_audio or no_video:
+            single_stream = 'none' in (dct.get(c) for c in ('acodec', 'vcodec'))
+            if single_stream and dct.get('ext'):
+                dct['container'] = dct['ext'] + '_dash'
+            if single_stream or itag == '17':
                 # avoid Youtube throttling
                 dct.update({
                     'protocol': 'http_dash_segments',
@@ -2067,8 +2064,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'downloader_options': {'http_chunk_size': CHUNK_SIZE}  # No longer useful?
                 })
 
-                if dct.get('ext'):
-                    dct['container'] = dct['ext'] + '_dash'
             formats.append(dct)
 
         hls_manifest_url = streaming_data.get('hlsManifestUrl')

From cdf40b6aa651d949ce01e9bec1a11f792e8af899 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 3 Apr 2023 21:07:10 +0100
Subject: [PATCH 625/860] [test] Update tests for Ubuntu 20.04 * 18.04 test
 runner was withdrawn * for now, disable Py 3.3/3.4 tests

---
 .github/workflows/ci.yml | 24 +++++++++++++-----------
 1 file changed, 13 insertions(+), 11 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a609f3704..51abdce1d 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -7,9 +7,10 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: [ubuntu-18.04]
+        os: [ubuntu-20.04]
         # TODO: python 2.6
-        python-version: [2.7, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, pypy-2.7, pypy-3.6, pypy-3.7]
+        # TODO: restore support for 3.3, 3.4
+        python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, pypy-2.7, pypy-3.6, pypy-3.7]
         python-impl: [cpython]
         ytdl-test-set: [core, download]
         run-tests-ext: [sh]
@@ -26,26 +27,27 @@ jobs:
           ytdl-test-set: download
           run-tests-ext: bat
         # jython
-        - os: ubuntu-18.04
+        - os: ubuntu-20.04
           python-impl: jython
           ytdl-test-set: core
           run-tests-ext: sh
-        - os: ubuntu-18.04
+        - os: ubuntu-20.04
           python-impl: jython
           ytdl-test-set: download
           run-tests-ext: sh
     steps:
-    - uses: actions/checkout@v2
-    - name: Set up Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v2
-      if: ${{ matrix.python-impl == 'cpython' }}
+    - uses: actions/checkout@v3
+    - name: Set up supported Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      if: ${{ matrix.python-impl == 'cpython' && ! contains(fromJSON('["3.3", "3.4"]'), matrix.python-version) }}
       with:
         python-version: ${{ matrix.python-version }}
     - name: Set up Java 8
       if: ${{ matrix.python-impl == 'jython' }}
-      uses: actions/setup-java@v1
+      uses: actions/setup-java@v2
       with:
         java-version: 8
+        distribution: 'zulu'
     - name: Install Jython
       if: ${{ matrix.python-impl == 'jython' }}
       run: |
@@ -70,9 +72,9 @@ jobs:
     name: Linter
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - name: Set up Python
-      uses: actions/setup-python@v2
+      uses: actions/setup-python@v4
       with:
         python-version: 3.9
     - name: Install flake8

From 557dbac173c30a51acd284b46f2d5460e539f51a Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 5 Apr 2023 18:29:24 +0100
Subject: [PATCH 626/860] [FragmentFD] Fix iteration with infinite limit

* fixes ytdl-org/youtube-dl/baa6c5e
* resolves #31885
---
 youtube_dl/downloader/dash.py | 22 +++++++++++-----------
 1 file changed, 11 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index cc30485f8..67a8e173f 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -1,5 +1,7 @@
 from __future__ import unicode_literals
 
+import itertools
+
 from .fragment import FragmentFD
 from ..compat import compat_urllib_error
 from ..utils import (
@@ -30,15 +32,13 @@ class DashSegmentsFD(FragmentFD):
         fragment_retries = self.params.get('fragment_retries', 0)
         skip_unavailable_fragments = self.params.get('skip_unavailable_fragments', True)
 
-        frag_index = 0
-        for i, fragment in enumerate(fragments):
-            frag_index += 1
+        for frag_index, fragment in enumerate(fragments, 1):
             if frag_index <= ctx['fragment_index']:
                 continue
             # In DASH, the first segment contains necessary headers to
             # generate a valid MP4 file, so always abort for the first segment
-            fatal = i == 0 or not skip_unavailable_fragments
-            for count in range(fragment_retries + 1):
+            fatal = frag_index == 1 or not skip_unavailable_fragments
+            for count in itertools.count():
                 try:
                     fragment_url = fragment.get('url')
                     if not fragment_url:
@@ -48,7 +48,6 @@ class DashSegmentsFD(FragmentFD):
                     if not success:
                         return False
                     self._append_fragment(ctx, frag_content)
-                    break
                 except compat_urllib_error.HTTPError as err:
                     # YouTube may often return 404 HTTP error for a fragment causing the
                     # whole download to fail. However if the same fragment is immediately
@@ -58,13 +57,14 @@ class DashSegmentsFD(FragmentFD):
                     # HTTP error.
                     if count < fragment_retries:
                         self.report_retry_fragment(err, frag_index, count + 1, fragment_retries)
+                        continue
                 except DownloadError:
                     # Don't retry fragment if error occurred during HTTP downloading
-                    # itself since it has own retry settings
-                    if not fatal:
-                        self.report_skip_fragment(frag_index)
-                        break
-                    raise
+                    # itself since it has its own retry settings
+                    if fatal:
+                        raise
+                    self.report_skip_fragment(frag_index)
+                break
 
             if count >= fragment_retries:
                 if not fatal:

From 78da22489b483988e198a8352893df9c6cf34032 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 5 Apr 2023 18:39:54 +0100
Subject: [PATCH 627/860] [compat] Add and use `compat_open()` like Py3
 `open()`

* resolves FIXME: ytdl-org/youtube-dl/commit/dfe5fa4
---
 youtube_dl/compat.py  | 11 +++++++++++
 youtube_dl/options.py |  6 ++----
 2 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 39551f810..fe62caf80 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -3127,6 +3127,16 @@ else:
         return ctypes.WINFUNCTYPE(*args, **kwargs)
 
 
+if sys.version_info < (3, 0):
+    # open(file, mode='r', buffering=- 1, encoding=None, errors=None, newline=None, closefd=True) not: opener=None
+    def compat_open(file_, *args, **kwargs):
+        if len(args) > 6 or 'opener' in kwargs:
+            raise ValueError('open: unsupported argument "opener"')
+        return io.open(file_, *args, **kwargs)
+else:
+    compat_open = open
+
+
 legacy = [
     'compat_HTMLParseError',
     'compat_HTMLParser',
@@ -3185,6 +3195,7 @@ __all__ = [
     'compat_kwargs',
     'compat_map',
     'compat_numeric_types',
+    'compat_open',
     'compat_ord',
     'compat_os_name',
     'compat_os_path_expanduser',
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index f6d2b0898..7b059b51e 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -11,6 +11,7 @@ from .compat import (
     compat_get_terminal_size,
     compat_getenv,
     compat_kwargs,
+    compat_open as open,
     compat_shlex_split,
 )
 from .utils import (
@@ -41,14 +42,11 @@ def _hide_login_info(opts):
 def parseOpts(overrideArguments=None):
     def _readOptions(filename_bytes, default=[]):
         try:
-            optionf = open(filename_bytes)
+            optionf = open(filename_bytes, encoding=preferredencoding())
         except IOError:
             return default  # silently skip if file is not present
         try:
-            # FIXME: https://github.com/ytdl-org/youtube-dl/commit/dfe5fa49aed02cf36ba9f743b11b0903554b5e56
             contents = optionf.read()
-            if sys.version_info < (3,):
-                contents = contents.decode(preferredencoding())
             res = compat_shlex_split(contents, comments=True)
         finally:
             optionf.close()

From 25124bd640acf2fbae71b2a52738ee41da548fb1 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 5 Apr 2023 18:47:49 +0100
Subject: [PATCH 628/860] [devscripts] Improve hack to convert command-line
 options to API options

* define equality for DateRange
* don't show default DateRange
---
 devscripts/cli_to_api.py | 25 ++++++++++++++++++++++---
 youtube_dl/utils.py      |  4 ++++
 2 files changed, 26 insertions(+), 3 deletions(-)

diff --git a/devscripts/cli_to_api.py b/devscripts/cli_to_api.py
index 2f4d6a458..9fb1d2ba8 100755
--- a/devscripts/cli_to_api.py
+++ b/devscripts/cli_to_api.py
@@ -49,15 +49,34 @@ def cli_to_api(*opts):
 
     # from https://github.com/yt-dlp/yt-dlp/issues/5859#issuecomment-1363938900
     default = parsed_options([])
-    diff = dict((k, v) for k, v in parsed_options(opts).items() if default[k] != v)
+
+    def neq_opt(a, b):
+        if a == b:
+            return False
+        if a is None and repr(type(object)).endswith(".utils.DateRange'>"):
+            return '0001-01-01 - 9999-12-31' != '{0}'.format(b)
+        return a != b
+
+    diff = dict((k, v) for k, v in parsed_options(opts).items() if neq_opt(default[k], v))
     if 'postprocessors' in diff:
         diff['postprocessors'] = [pp for pp in diff['postprocessors'] if pp not in default['postprocessors']]
     return diff
 
 
 def main():
-    from pprint import pprint
-    pprint(cli_to_api(*sys.argv))
+    from pprint import PrettyPrinter
+
+    pprint = PrettyPrinter()
+    super_format = pprint.format
+
+    def format(object, context, maxlevels, level):
+        if repr(type(object)).endswith(".utils.DateRange'>"):
+            return '{0}: {1}>'.format(repr(object)[:-2], object), True, False
+        return super_format(object, context, maxlevels, level)
+
+    pprint.format = format
+
+    pprint.pprint(cli_to_api(*sys.argv))
 
 
 if __name__ == '__main__':
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index f3c7af437..d80ceb007 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3190,6 +3190,10 @@ class DateRange(object):
     def __str__(self):
         return '%s - %s' % (self.start.isoformat(), self.end.isoformat())
 
+    def __eq__(self, other):
+        return (isinstance(other, DateRange)
+                and self.start == other.start and self.end == other.end)
+
 
 def platform_name():
     """ Returns the platform name as a compat_str """

From 9f4d83ff4255d8840c0fa9b367722c129ebecdb2 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 5 Apr 2023 18:50:25 +0100
Subject: [PATCH 629/860] [options] Add --mtime option, unsets default
 --no-mtime

* resolves #1709 (!)
---
 youtube_dl/options.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 7b059b51e..d802b7e59 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -731,9 +731,13 @@ def parseOpts(overrideArguments=None):
         '--no-part',
         action='store_true', dest='nopart', default=False,
         help='Do not use .part files - write directly into output file')
+    filesystem.add_option(
+        '--mtime',
+        action='store_true', dest='updatetime', default=True,
+        help='Use the Last-modified header to set the file modification time (default)')
     filesystem.add_option(
         '--no-mtime',
-        action='store_false', dest='updatetime', default=True,
+        action='store_false', dest='updatetime',
         help='Do not use the Last-modified header to set the file modification time')
     filesystem.add_option(
         '--write-description',

From d6ae3b77cd50083ef245c28f904ee0b70a77d5c6 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 6 Apr 2023 14:11:18 +0100
Subject: [PATCH 630/860] [core] Avoid deepcopy of ctx dict (fix f35b757)

* may now contain `LazyList`s
* resolves #31999
---
 youtube_dl/YoutubeDL.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index bcf781744..2c0d4926c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1399,7 +1399,7 @@ class YoutubeDL(object):
             filters = [self._build_format_filter(f) for f in selector.filters]
 
             def final_selector(ctx):
-                ctx_copy = copy.deepcopy(ctx)
+                ctx_copy = dict(ctx)
                 for _filter in filters:
                     ctx_copy['formats'] = list(filter(_filter, ctx_copy['formats']))
                 return selector_function(ctx_copy)

From f8253a528935f78e1a3b724db8c1f0089f99314a Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 6 Apr 2023 19:42:36 +0100
Subject: [PATCH 631/860] [core] Avoid deepcopy of ctx dict (fix f35b757) (Pt
 2)

---
 youtube_dl/YoutubeDL.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 2c0d4926c..927b19417 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1389,11 +1389,10 @@ class YoutubeDL(object):
                         'abr': formats_info[1].get('abr'),
                         'ext': output_ext,
                     }
-                video_selector, audio_selector = map(_build_selector_function, selector.selector)
 
                 def selector_function(ctx):
-                    for pair in itertools.product(
-                            video_selector(copy.deepcopy(ctx)), audio_selector(copy.deepcopy(ctx))):
+                    selector_fn = lambda x: _build_selector_function(x)(ctx)
+                    for pair in itertools.product(*map(selector_fn, selector.selector)):
                         yield _merge(pair)
 
             filters = [self._build_format_filter(f) for f in selector.filters]

From 213d1d91bfc4a00fefc72fa2730555d51060b42d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 6 Apr 2023 19:49:46 +0100
Subject: [PATCH 632/860] [core] No longer importing copy

---
 youtube_dl/YoutubeDL.py | 1 -
 1 file changed, 1 deletion(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 927b19417..2a1e59bf8 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -5,7 +5,6 @@ from __future__ import absolute_import, unicode_literals
 
 import collections
 import contextlib
-import copy
 import datetime
 import errno
 import fileinput

From fe7e13066c20b10fe48bc154431440da36baec53 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 10 Apr 2023 17:12:31 +0100
Subject: [PATCH 633/860] [core] Add and use sanitize_info() method from yt-dlp

---
 youtube_dl/YoutubeDL.py | 38 ++++++++++++++++++++++++++++++++++----
 1 file changed, 34 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 2a1e59bf8..2719d546f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -30,9 +30,12 @@ from string import ascii_letters
 from .compat import (
     compat_basestring,
     compat_cookiejar,
+    compat_filter as filter,
     compat_get_terminal_size,
     compat_http_client,
+    compat_integer_types,
     compat_kwargs,
+    compat_map as map,
     compat_numeric_types,
     compat_os_name,
     compat_str,
@@ -64,6 +67,7 @@ from .utils import (
     int_or_none,
     ISO3166Utils,
     locked_file,
+    LazyList,
     make_HTTPS_handler,
     MaxDownloadsReached,
     orderedSet,
@@ -2109,10 +2113,36 @@ class YoutubeDL(object):
         return self._download_retcode
 
     @staticmethod
-    def filter_requested_info(info_dict):
-        return dict(
-            (k, v) for k, v in info_dict.items()
-            if k not in ['requested_formats', 'requested_subtitles'])
+    def sanitize_info(info_dict, remove_private_keys=False):
+        ''' Sanitize the infodict for converting to json '''
+        if info_dict is None:
+            return info_dict
+
+        if remove_private_keys:
+            reject = lambda k, v: (v is None
+                                   or k.startswith('__')
+                                   or k in ('requested_formats',
+                                            'requested_subtitles'))
+        else:
+            reject = lambda k, v: False
+
+        def filter_fn(obj):
+            if isinstance(obj, dict):
+                return dict((k, filter_fn(v)) for k, v in obj.items() if not reject(k, v))
+            elif isinstance(obj, (list, tuple, set, LazyList)):
+                return list(map(filter_fn, obj))
+            elif obj is None or any(isinstance(obj, c)
+                                    for c in (compat_integer_types,
+                                              (compat_str, float, bool))):
+                return obj
+            else:
+                return repr(obj)
+
+        return filter_fn(info_dict)
+
+    @classmethod
+    def filter_requested_info(cls, info_dict):
+        return cls.sanitize_info(info_dict, True)
 
     def post_process(self, filename, ie_info):
         """Run all the postprocessors on the given file."""

From 735e87adfc44b284dcdb4d9a0155ce0616e3af97 Mon Sep 17 00:00:00 2001
From: Gabriel Nagy <gabrielnagy@me.com>
Date: Thu, 13 Apr 2023 01:40:38 +0300
Subject: [PATCH 634/860] [core] Sanitize info dict before dumping JSON (fixes
 fe7e130)  (#32032)

* follow up to fe7e130 which didn't fix everything.

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/YoutubeDL.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 2719d546f..117f1c513 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1777,7 +1777,7 @@ class YoutubeDL(object):
             self.to_stdout(formatSeconds(info_dict['duration']))
         print_mandatory('format')
         if self.params.get('forcejson', False):
-            self.to_stdout(json.dumps(info_dict))
+            self.to_stdout(json.dumps(self.sanitize_info(info_dict)))
 
     def process_info(self, info_dict):
         """Process a single resolved IE result."""
@@ -2091,7 +2091,7 @@ class YoutubeDL(object):
                 raise
             else:
                 if self.params.get('dump_single_json', False):
-                    self.to_stdout(json.dumps(res))
+                    self.to_stdout(json.dumps(self.sanitize_info(res)))
 
         return self._download_retcode
 
@@ -2100,6 +2100,7 @@ class YoutubeDL(object):
                 [info_filename], mode='r',
                 openhook=fileinput.hook_encoded('utf-8'))) as f:
             # FileInput doesn't have a read method, we can't call json.load
+            # TODO: let's use io.open(), then
             info = self.filter_requested_info(json.loads('\n'.join(f)))
         try:
             self.process_ie_result(info, download=True)

From 2da3fa04a68ff0652f49d6874d82b7a0edb85ea3 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 11 Apr 2023 17:36:27 +0100
Subject: [PATCH 635/860] [YouTube] Simplify signature patterns

---
 youtube_dl/extractor/youtube.py | 8 +++-----
 1 file changed, 3 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ae3416b20..80fff7ada 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -19,6 +19,7 @@ from ..compat import (
     compat_urllib_parse_parse_qs as compat_parse_qs,
     compat_urllib_parse_unquote_plus,
     compat_urllib_parse_urlparse,
+    compat_zip as zip,
 )
 from ..jsinterp import JSInterpreter
 from ..utils import (
@@ -1555,17 +1556,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
              r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
-             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\);[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\)',
-             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
+             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\))?',
              r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
              # Obsolete patterns
-             r'(["\'])signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
+             r'("|\')signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'\bc\s*&&\s*a\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
             jscode, 'Initial JS player signature function name', group='sig')
 

From 26035bde46c0acc30dc053618451d9aeca4b7709 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 13 Apr 2023 00:15:07 +0100
Subject: [PATCH 636/860] [DashSegmentsFD] Correctly detect errors when
 `fragment_retries` == 0

* use the success flag instead of the retry count
* establish the fragment_url outside the retry loop
* only report skipping a fragment once.
* resolves #32033
---
 youtube_dl/downloader/dash.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 67a8e173f..2800d4260 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -38,12 +38,13 @@ class DashSegmentsFD(FragmentFD):
             # In DASH, the first segment contains necessary headers to
             # generate a valid MP4 file, so always abort for the first segment
             fatal = frag_index == 1 or not skip_unavailable_fragments
+            fragment_url = fragment.get('url')
+            if not fragment_url:
+                assert fragment_base_url
+                fragment_url = urljoin(fragment_base_url, fragment['path'])
+            success = False
             for count in itertools.count():
                 try:
-                    fragment_url = fragment.get('url')
-                    if not fragment_url:
-                        assert fragment_base_url
-                        fragment_url = urljoin(fragment_base_url, fragment['path'])
                     success, frag_content = self._download_fragment(ctx, fragment_url, info_dict)
                     if not success:
                         return False
@@ -63,14 +64,13 @@ class DashSegmentsFD(FragmentFD):
                     # itself since it has its own retry settings
                     if fatal:
                         raise
-                    self.report_skip_fragment(frag_index)
                 break
 
-            if count >= fragment_retries:
+            if not success:
                 if not fatal:
                     self.report_skip_fragment(frag_index)
                     continue
-                self.report_error('giving up after %s fragment retries' % fragment_retries)
+                self.report_error('giving up after %s fragment retries' % count)
                 return False
 
         self._finish_frag_download(ctx)

From 211cbfd5d46025a8e4d8f9f3d424aaada4698974 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 21 Apr 2023 14:04:30 +0100
Subject: [PATCH 637/860] [jsinterp] Minimally handle arithmetic operator
 precedence

Resolves #32066
---
 test/test_jsinterp.py  | 11 +++++++++++
 youtube_dl/jsinterp.py | 40 +++++++++++++++++++++++++++++++++++++---
 2 files changed, 48 insertions(+), 3 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 5d129433d..e121358d7 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -505,6 +505,17 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('function x(){return 1236566549 << 5}')
         self.assertEqual(jsi.call_function('x'), 915423904)
 
+    def test_32066(self):
+        jsi = JSInterpreter("function x(){return Math.pow(3, 5) + new Date('1970-01-01T08:01:42.000+08:00') / 1000 * -239 - -24205;}")
+        self.assertEqual(jsi.call_function('x'), 70)
+
+    def test_unary_operators(self):
+        jsi = JSInterpreter('function f(){return 2  -  - - 2;}')
+        self.assertEqual(jsi.call_function('f'), 0)
+        # fails
+        # jsi = JSInterpreter('function f(){return 2 + - + - - 2;}')
+        # self.assertEqual(jsi.call_function('f'), 0)
+
     """ # fails so far
     def test_packed(self):
         jsi = JSInterpreter('''function x(p,a,c,k,e,d){while(c--)if(k[c])p=p.replace(new RegExp('\\b'+c.toString(a)+'\\b','g'),k[c]);return p}''')
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index ab7d6f926..a06fc4ff5 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -1,5 +1,6 @@
 from __future__ import unicode_literals
 
+from functools import update_wrapper
 import itertools
 import json
 import math
@@ -23,11 +24,23 @@ from .compat import (
 )
 
 
+def wraps_op(op):
+
+    def update_and_rename_wrapper(w):
+        f = update_wrapper(w, op)
+        # fn names are str in both Py 2/3
+        f.__name__ = str('JS_') + f.__name__
+        return f
+
+    return update_and_rename_wrapper
+
+
 def _js_bit_op(op):
 
     def zeroise(x):
         return 0 if x in (None, JS_Undefined) else x
 
+    @wraps_op(op)
     def wrapped(a, b):
         return op(zeroise(a), zeroise(b)) & 0xffffffff
 
@@ -36,6 +49,7 @@ def _js_bit_op(op):
 
 def _js_arith_op(op):
 
+    @wraps_op(op)
     def wrapped(a, b):
         if JS_Undefined in (a, b):
             return float('nan')
@@ -66,6 +80,7 @@ def _js_exp(a, b):
 
 def _js_eq_op(op):
 
+    @wraps_op(op)
     def wrapped(a, b):
         if set((a, b)) <= set((None, JS_Undefined)):
             return op(a, a)
@@ -76,6 +91,7 @@ def _js_eq_op(op):
 
 def _js_comp_op(op):
 
+    @wraps_op(op)
     def wrapped(a, b):
         if JS_Undefined in (a, b):
             return False
@@ -356,6 +372,7 @@ class JSInterpreter(object):
             return right_val
 
         try:
+            # print('Eval:', opfunc.__name__, left_val, right_val)
             return opfunc(left_val, right_val)
         except Exception as e:
             raise self.Exception('Failed to evaluate {left_val!r:.50} {op} {right_val!r:.50}'.format(**locals()), expr, cause=e)
@@ -395,6 +412,7 @@ class JSInterpreter(object):
             raise self.Exception('Recursion limit reached')
         allow_recursion -= 1
 
+        # print('At: ' + stmt[:60])
         should_return = False
         # fails on (eg) if (...) stmt1; else stmt2;
         sub_statements = list(self._separate(stmt, ';')) or ['']
@@ -702,9 +720,24 @@ class JSInterpreter(object):
                 continue
 
             right_expr = separated.pop()
-            while op == '-' and len(separated) > 1 and not separated[-1].strip():
-                right_expr = '-' + right_expr
-                separated.pop()
+            # handle operators that are both unary and binary, minimal BODMAS
+            if op in ('+', '-'):
+                undone = 0
+                while len(separated) > 1 and not separated[-1].strip():
+                    undone += 1
+                    separated.pop()
+                if op == '-' and undone % 2 != 0:
+                    right_expr = op + right_expr
+                left_val = separated[-1]
+                for dm_op in ('*', '%', '/', '**'):
+                    bodmas = tuple(self._separate(left_val, dm_op, skip_delims=skip_delim))
+                    if len(bodmas) > 1 and not bodmas[-1].strip():
+                        expr = op.join(separated) + op + right_expr
+                        right_expr = None
+                        break
+                if right_expr is None:
+                    continue
+
             left_val = self.interpret_expression(op.join(separated), local_vars, allow_recursion)
             return self._operator(op, left_val, right_expr, expr, local_vars, allow_recursion), should_return
 
@@ -955,6 +988,7 @@ class JSInterpreter(object):
     def build_function(self, argnames, code, *global_stack):
         global_stack = list(global_stack) or [{}]
         argnames = tuple(argnames)
+        # import pdb; pdb.set_trace()
 
         def resf(args, kwargs={}, allow_recursion=100):
             global_stack[0].update(

From 64d6dd64c8b7a35a87655d27fc83f2e98ef6ce13 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 23 Apr 2023 22:58:35 +0100
Subject: [PATCH 638/860] [YouTube] Support Releases tab

---
 youtube_dl/extractor/youtube.py | 114 +++++++++++++++++++-------------
 youtube_dl/utils.py             |   9 ++-
 2 files changed, 74 insertions(+), 49 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 80fff7ada..0411c49f1 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -31,6 +31,7 @@ from ..utils import (
     extract_attributes,
     get_element_by_attribute,
     int_or_none,
+    join_nonempty,
     js_to_json,
     LazyList,
     merge_dicts,
@@ -45,6 +46,7 @@ from ..utils import (
     str_to_int,
     traverse_obj,
     try_get,
+    txt_or_none,
     unescapeHTML,
     unified_strdate,
     unsmuggle_url,
@@ -2608,6 +2610,17 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'uploader_id': '@lexwill718',
         },
         'playlist_mincount': 75,
+    }, {
+        # Releases tab
+        'url': 'https://www.youtube.com/@daftpunk/releases',
+        'info_dict': {
+            'id': 'UC_kRDKYrUlrbtrSiyu5Tflg',
+            'title': 'Daft Punk - Releases',
+            'description': 'Daft Punk (1993 - 2021) - Official YouTube Channel',
+            'uploader_id': '@daftpunk',
+            'uploader': 'Daft Punk',
+        },
+        'playlist_mincount': 36,
     }, {
         'url': 'https://invidio.us/channel/UCmlqkdCBesrv2Lak1mF_MxA',
         'only_matching': True,
@@ -2822,6 +2835,12 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                 continue
             return renderer
 
+    @staticmethod
+    def _get_text(r, k):
+        return traverse_obj(
+            r, (k, 'runs', 0, 'text'), (k, 'simpleText'),
+            expected_type=txt_or_none)
+
     def _grid_entries(self, grid_renderer):
         for item in grid_renderer['items']:
             if not isinstance(item, dict):
@@ -2829,9 +2848,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             renderer = self._extract_grid_item_renderer(item)
             if not isinstance(renderer, dict):
                 continue
-            title = try_get(
-                renderer, (lambda x: x['title']['runs'][0]['text'],
-                           lambda x: x['title']['simpleText']), compat_str)
+            title = self._get_text(renderer, 'title')
             # playlist
             playlist_id = renderer.get('playlistId')
             if playlist_id:
@@ -2848,8 +2865,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             # channel
             channel_id = renderer.get('channelId')
             if channel_id:
-                title = try_get(
-                    renderer, lambda x: x['title']['simpleText'], compat_str)
+                title = self._get_text(renderer, 'title')
                 yield self.url_result(
                     'https://www.youtube.com/channel/%s' % channel_id,
                     ie=YoutubeTabIE.ie_key(), video_title=title)
@@ -2958,15 +2974,26 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
 
     def _rich_grid_entries(self, contents):
         for content in contents:
-            video_renderer = try_get(
-                content,
-                (lambda x: x['richItemRenderer']['content']['videoRenderer'],
-                 lambda x: x['richItemRenderer']['content']['reelItemRenderer']),
-                dict)
+            content = traverse_obj(
+                content, ('richItemRenderer', 'content'),
+                expected_type=dict) or {}
+            video_renderer = traverse_obj(
+                content, 'videoRenderer', 'reelItemRenderer',
+                expected_type=dict)
             if video_renderer:
                 entry = self._video_entry(video_renderer)
                 if entry:
                     yield entry
+            # playlist
+            renderer = traverse_obj(
+                content, 'playlistRenderer', expected_type=dict) or {}
+            title = self._get_text(renderer, 'title')
+            playlist_id = renderer.get('playlistId')
+            if playlist_id:
+                yield self.url_result(
+                    'https://www.youtube.com/playlist?list=%s' % playlist_id,
+                    ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
+                    video_title=title)
 
     @staticmethod
     def _build_continuation_query(continuation, ctp=None):
@@ -3071,6 +3098,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                 return
             for entry in self._rich_grid_entries(rich_grid_renderer.get('contents') or []):
                 yield entry
+
             continuation = self._extract_continuation(rich_grid_renderer)
 
         ytcfg = self._extract_ytcfg(item_id, webpage)
@@ -3213,50 +3241,41 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
         uploader['channel'] = uploader['uploader']
         return uploader
 
-    @staticmethod
-    def _extract_alert(data):
+    @classmethod
+    def _extract_alert(cls, data):
         alerts = []
-        for alert in try_get(data, lambda x: x['alerts'], list) or []:
-            if not isinstance(alert, dict):
-                continue
-            alert_text = try_get(
-                alert, lambda x: x['alertRenderer']['text'], dict)
+        for alert in traverse_obj(data, ('alerts', Ellipsis), expected_type=dict):
+            alert_text = traverse_obj(
+                alert, (None, lambda x: x['alertRenderer']['text']), get_all=False)
             if not alert_text:
                 continue
-            text = try_get(
-                alert_text,
-                (lambda x: x['simpleText'], lambda x: x['runs'][0]['text']),
-                compat_str)
+            text = cls._get_text(alert_text, 'text')
             if text:
                 alerts.append(text)
         return '\n'.join(alerts)
 
     def _extract_from_tabs(self, item_id, webpage, data, tabs):
         selected_tab = self._extract_selected_tab(tabs)
-        renderer = try_get(
-            data, lambda x: x['metadata']['channelMetadataRenderer'], dict)
+        renderer = traverse_obj(data, ('metadata', 'channelMetadataRenderer'),
+                                expected_type=dict) or {}
         playlist_id = item_id
         title = description = None
         if renderer:
-            channel_title = renderer.get('title') or item_id
-            tab_title = selected_tab.get('title')
-            title = channel_title or item_id
-            if tab_title:
-                title += ' - %s' % tab_title
-            if selected_tab.get('expandedText'):
-                title += ' - %s' % selected_tab['expandedText']
-            description = renderer.get('description')
-            playlist_id = renderer.get('externalId')
+            channel_title = txt_or_none(renderer.get('title')) or item_id
+            tab_title = txt_or_none(selected_tab.get('title'))
+            title = join_nonempty(
+                channel_title or item_id, tab_title,
+                txt_or_none(selected_tab.get('expandedText')),
+                delim=' - ')
+            description = txt_or_none(renderer.get('description'))
+            playlist_id = txt_or_none(renderer.get('externalId')) or playlist_id
         else:
-            renderer = try_get(
-                data, lambda x: x['metadata']['playlistMetadataRenderer'], dict)
-            if renderer:
-                title = renderer.get('title')
-            else:
-                renderer = try_get(
-                    data, lambda x: x['header']['hashtagHeaderRenderer'], dict)
-                if renderer:
-                    title = try_get(renderer, lambda x: x['hashtag']['simpleText'])
+            renderer = traverse_obj(data,
+                                    ('metadata', 'playlistMetadataRenderer'),
+                                    ('header', 'hashtagHeaderRenderer'),
+                                    expected_type=dict) or {}
+            title = traverse_obj(renderer, 'title', ('hashtag', 'simpleText'),
+                                 expected_type=txt_or_none)
         playlist = self.playlist_result(
             self._entries(selected_tab, item_id, webpage),
             playlist_id=playlist_id, playlist_title=title,
@@ -3264,15 +3283,16 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
         return merge_dicts(playlist, self._extract_uploader(renderer, data))
 
     def _extract_from_playlist(self, item_id, url, data, playlist):
-        title = playlist.get('title') or try_get(
-            data, lambda x: x['titleText']['simpleText'], compat_str)
-        playlist_id = playlist.get('playlistId') or item_id
+        title = traverse_obj((playlist, data),
+                             (0, 'title'), (1, 'titleText', 'simpleText'),
+                             expected_type=txt_or_none)
+        playlist_id = txt_or_none(playlist.get('playlistId')) or item_id
         # Inline playlist rendition continuation does not always work
         # at Youtube side, so delegating regular tab-based playlist URL
         # processing whenever possible.
-        playlist_url = urljoin(url, try_get(
-            playlist, lambda x: x['endpoint']['commandMetadata']['webCommandMetadata']['url'],
-            compat_str))
+        playlist_url = urljoin(url, traverse_obj(
+            playlist, ('endpoint', 'commandMetadata', 'webCommandMetadata', 'url'),
+            expected_type=url_or_none))
         if playlist_url and playlist_url != url:
             return self.url_result(
                 playlist_url, ie=YoutubeTabIE.ie_key(), video_id=playlist_id,
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d80ceb007..65ddb3b0f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3753,6 +3753,11 @@ def strip_or_none(v, default=None):
     return v.strip() if isinstance(v, compat_str) else default
 
 
+def txt_or_none(v, default=None):
+    """ Combine str/strip_or_none, disallow blank value (for traverse_obj) """
+    return default if v is None else (compat_str(v).strip() or default)
+
+
 def url_or_none(url):
     if not url or not isinstance(url, compat_str):
         return None
@@ -4096,8 +4101,8 @@ def escape_url(url):
     ).geturl()
 
 
-def parse_qs(url):
-    return compat_parse_qs(compat_urllib_parse.urlparse(url).query)
+def parse_qs(url, **kwargs):
+    return compat_parse_qs(compat_urllib_parse.urlparse(url).query, **kwargs)
 
 
 def read_batch_urls(batch_fd):

From 11cc3f3ad03a88d6cb1eab18a8e5dd6bf148ac54 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 11 May 2023 20:53:07 +0100
Subject: [PATCH 639/860] [utils] Fix `compiled_regex_type` in 249f2b6

---
 youtube_dl/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 65ddb3b0f..584581b6a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -56,6 +56,7 @@ from .compat import (
     compat_kwargs,
     compat_os_name,
     compat_re_Match,
+    compat_re_Pattern,
     compat_shlex_quote,
     compat_str,
     compat_struct_pack,
@@ -86,7 +87,7 @@ def register_socks_protocols():
 
 
 # Unfavoured alias
-compiled_regex_type = compat_re_Match
+compiled_regex_type = compat_re_Pattern
 
 
 def random_user_agent():

From a85a875fef2e9b097c3f6f93f1d0cead06f84e43 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 11 May 2023 20:59:30 +0100
Subject: [PATCH 640/860] [jsinterp] Handle NaN in bitwise operators * also add
 _NaN * also pull function naming from yt-dlp

---
 test/test_jsinterp.py  | 11 +++++++++++
 youtube_dl/jsinterp.py | 41 ++++++++++++++++++++++++++++++++---------
 2 files changed, 43 insertions(+), 9 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index e121358d7..a8f312fde 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -18,6 +18,7 @@ class TestJSInterpreter(unittest.TestCase):
     def test_basic(self):
         jsi = JSInterpreter('function x(){;}')
         self.assertEqual(jsi.call_function('x'), None)
+        self.assertEqual(repr(jsi.extract_function('x')), 'F<x>')
 
         jsi = JSInterpreter('function x3(){return 42;}')
         self.assertEqual(jsi.call_function('x3'), 42)
@@ -505,6 +506,16 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('function x(){return 1236566549 << 5}')
         self.assertEqual(jsi.call_function('x'), 915423904)
 
+    def test_bitwise_operators_madness(self):
+        jsi = JSInterpreter('function x(){return null << 5}')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+        jsi = JSInterpreter('function x(){return undefined >> 5}')
+        self.assertEqual(jsi.call_function('x'), 0)
+
+        jsi = JSInterpreter('function x(){return 42 << NaN}')
+        self.assertEqual(jsi.call_function('x'), 42)
+
     def test_32066(self):
         jsi = JSInterpreter("function x(){return Math.pow(3, 5) + new Date('1970-01-01T08:01:42.000+08:00') / 1000 * -239 - -24205;}")
         self.assertEqual(jsi.call_function('x'), 70)
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index a06fc4ff5..bb406647a 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -1,12 +1,13 @@
 from __future__ import unicode_literals
 
-from functools import update_wrapper
 import itertools
 import json
 import math
 import operator
 import re
 
+from functools import update_wrapper
+
 from .utils import (
     error_to_compat_str,
     ExtractorError,
@@ -24,6 +25,22 @@ from .compat import (
 )
 
 
+# name JS functions
+class function_with_repr(object):
+    # from yt_dlp/utils.py, but in this module
+    # repr_ is always set
+    def __init__(self, func, repr_):
+        update_wrapper(self, func)
+        self.func, self.__repr = func, repr_
+
+    def __call__(self, *args, **kwargs):
+        return self.func(*args, **kwargs)
+
+    def __repr__(self):
+        return self.__repr
+
+
+# name JS operators
 def wraps_op(op):
 
     def update_and_rename_wrapper(w):
@@ -35,10 +52,13 @@ def wraps_op(op):
     return update_and_rename_wrapper
 
 
+_NaN = float('nan')
+
+
 def _js_bit_op(op):
 
     def zeroise(x):
-        return 0 if x in (None, JS_Undefined) else x
+        return 0 if x in (None, JS_Undefined, _NaN) else x
 
     @wraps_op(op)
     def wrapped(a, b):
@@ -52,7 +72,7 @@ def _js_arith_op(op):
     @wraps_op(op)
     def wrapped(a, b):
         if JS_Undefined in (a, b):
-            return float('nan')
+            return _NaN
         return op(a or 0, b or 0)
 
     return wrapped
@@ -60,13 +80,13 @@ def _js_arith_op(op):
 
 def _js_div(a, b):
     if JS_Undefined in (a, b) or not (a and b):
-        return float('nan')
+        return _NaN
     return operator.truediv(a or 0, b) if b else float('inf')
 
 
 def _js_mod(a, b):
     if JS_Undefined in (a, b) or not b:
-        return float('nan')
+        return _NaN
     return (a or 0) % b
 
 
@@ -74,7 +94,7 @@ def _js_exp(a, b):
     if not b:
         return 1  # even 0 ** 0 !!
     elif JS_Undefined in (a, b):
-        return float('nan')
+        return _NaN
     return (a or 0) ** b
 
 
@@ -285,6 +305,8 @@ class JSInterpreter(object):
     def _named_object(self, namespace, obj):
         self.__named_object_counter += 1
         name = '%s%d' % (self._OBJ_NAME, self.__named_object_counter)
+        if callable(obj) and not isinstance(obj, function_with_repr):
+            obj = function_with_repr(obj, 'F<%s>' % (self.__named_object_counter, ))
         namespace[name] = obj
         return name
 
@@ -693,7 +715,7 @@ class JSInterpreter(object):
         elif expr == 'undefined':
             return JS_Undefined, should_return
         elif expr == 'NaN':
-            return float('NaN'), should_return
+            return _NaN, should_return
 
         elif md.get('return'):
             return local_vars[m.group('name')], should_return
@@ -953,7 +975,9 @@ class JSInterpreter(object):
         return self.build_arglist(func_m.group('args')), code
 
     def extract_function(self, funcname):
-        return self.extract_function_from_code(*self.extract_function_code(funcname))
+        return function_with_repr(
+            self.extract_function_from_code(*self.extract_function_code(funcname)),
+            'F<%s>' % (funcname, ))
 
     def extract_function_from_code(self, argnames, code, *global_stack):
         local_vars = {}
@@ -988,7 +1012,6 @@ class JSInterpreter(object):
     def build_function(self, argnames, code, *global_stack):
         global_stack = list(global_stack) or [{}]
         argnames = tuple(argnames)
-        # import pdb; pdb.set_trace()
 
         def resf(args, kwargs={}, allow_recursion=100):
             global_stack[0].update(

From 6ed34338285f722d0da312ce0af3a15a077a3e2a Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 11 May 2023 21:02:01 +0100
Subject: [PATCH 641/860] [jsinterp] Add short-cut evaluation for common
 expression * special handling for (d%e.length+e.length)%e.length speeds up
 ~6%

---
 youtube_dl/jsinterp.py | 22 ++++++++++++++++++++--
 1 file changed, 20 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index bb406647a..f837865c4 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -502,8 +502,15 @@ class JSInterpreter(object):
                 expr = self._dump(inner, local_vars) + outer
 
         if expr.startswith('('):
-            inner, outer = self._separate_at_paren(expr)
-            inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion)
+
+            m = re.match(r'\((?P<d>[a-z])%(?P<e>[a-z])\.length\+(?P=e)\.length\)%(?P=e)\.length', expr)
+            if m:
+                # short-cut eval of frequently used `(d%e.length+e.length)%e.length`, worth ~6% on `pytest -k test_nsig`
+                outer = None
+                inner, should_abort = self._offset_e_by_d(m.group('d'), m.group('e'), local_vars)
+            else:
+                inner, outer = self._separate_at_paren(expr)
+                inner, should_abort = self.interpret_statement(inner, local_vars, allow_recursion)
             if not outer or should_abort:
                 return inner, should_abort or should_return
             else:
@@ -957,6 +964,17 @@ class JSInterpreter(object):
 
         return obj
 
+    @staticmethod
+    def _offset_e_by_d(d, e, local_vars):
+        """ Short-cut eval: (d%e.length+e.length)%e.length """
+        try:
+            d = local_vars[d]
+            e = local_vars[e]
+            e = len(e)
+            return _js_mod(_js_mod(d, e) + e, e), False
+        except Exception:
+            return None, True
+
     def extract_function_code(self, funcname):
         """ @returns argnames, code """
         func_m = re.search(

From d1c6c5c4d618fa950813c0c71aede34a5ac851e9 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 11 May 2023 21:17:31 +0100
Subject: [PATCH 642/860] [core] Improve platform debug log, based on yt-dlp

---
 youtube_dl/YoutubeDL.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 117f1c513..212c04298 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -25,6 +25,7 @@ import tokenize
 import traceback
 import random
 
+from ssl import OPENSSL_VERSION
 from string import ascii_letters
 
 from .compat import (
@@ -66,6 +67,7 @@ from .utils import (
     HEADRequest,
     int_or_none,
     ISO3166Utils,
+    join_nonempty,
     locked_file,
     LazyList,
     make_HTTPS_handler,
@@ -2395,9 +2397,20 @@ class YoutubeDL(object):
                 return impl_name + ' version %d.%d.%d' % sys.pypy_version_info[:3]
             return impl_name
 
-        self._write_string('[debug] Python version %s (%s) - %s\n' % (
-            platform.python_version(), python_implementation(),
-            platform_name()))
+        def libc_ver():
+            try:
+                return platform.libc_ver()
+            except OSError:  # We may not have access to the executable
+                return []
+
+        self._write_string('[debug] Python %s (%s %s) - %s (%s%s)\n' % (
+            platform.python_version(),
+            python_implementation(),
+            platform.architecture()[0],
+            platform_name(),
+            OPENSSL_VERSION,
+            ', %s' % (join_nonempty(*libc_ver(), delim=' ') or '-'),
+        ))
 
         exe_versions = FFmpegPostProcessor.get_versions(self)
         exe_versions['rtmpdump'] = rtmpdump_version()

From d89c2137ba4c1def185358a9ff48642e05ac65a2 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 19 May 2023 13:09:18 +0100
Subject: [PATCH 643/860] [jsinterp] Small updates for a85a875 * update
 signature tests * clarify NaN handling

---
 test/test_jsinterp.py          |  3 +++
 test/test_youtube_signature.py |  8 ++++++++
 youtube_dl/jsinterp.py         | 12 +++++-------
 3 files changed, 16 insertions(+), 7 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index a8f312fde..1cc148b15 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -516,6 +516,9 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('function x(){return 42 << NaN}')
         self.assertEqual(jsi.call_function('x'), 42)
 
+        jsi = JSInterpreter('function x(){return 42 << Infinity}')
+        self.assertEqual(jsi.call_function('x'), 42)
+
     def test_32066(self):
         jsi = JSInterpreter("function x(){return Math.pow(3, 5) + new Date('1970-01-01T08:01:42.000+08:00') / 1000 * -239 - -24205;}")
         self.assertEqual(jsi.call_function('x'), 70)
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index decf7ee38..d41d708a0 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -143,6 +143,14 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/dac945fd/player_ias.vflset/en_US/base.js',
         'o8BkRxXhuYsBCWi6RplPdP', '3Lx32v_hmzTm6A',
     ),
+    (
+        'https://www.youtube.com/s/player/6f20102c/player_ias.vflset/en_US/base.js',
+        'lE8DhoDmKqnmJJ', 'pJTTX6XyJP2BYw',
+    ),
+    (
+        'https://www.youtube.com/s/player/cfa9e7cb/player_ias.vflset/en_US/base.js',
+        'qO0NiMtYQ7TeJnfFG2', 'k9cuJDHNS5O7kQ',
+    ),
 ]
 
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index f837865c4..dc580943e 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -2,7 +2,6 @@ from __future__ import unicode_literals
 
 import itertools
 import json
-import math
 import operator
 import re
 
@@ -52,6 +51,10 @@ def wraps_op(op):
     return update_and_rename_wrapper
 
 
+# NB In principle NaN cannot be checked by membership.
+# Here all NaN values are actually this one, so _NaN is _NaN,
+# although _NaN != _NaN.
+
 _NaN = float('nan')
 
 
@@ -126,13 +129,8 @@ def _js_comp_op(op):
 
 def _js_ternary(cndn, if_true=True, if_false=False):
     """Simulate JS's ternary operator (cndn?if_true:if_false)"""
-    if cndn in (False, None, 0, '', JS_Undefined):
+    if cndn in (False, None, 0, '', JS_Undefined, _NaN):
         return if_false
-    try:
-        if math.isnan(cndn):  # NB: NaN cannot be checked by membership
-            return if_false
-    except TypeError:
-        pass
     return if_true
 
 

From 1f7c6f8b2ba5bedc9b4da279659688fbbf06a059 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 19 May 2023 13:12:59 +0100
Subject: [PATCH 644/860] [core] Further improve platform debug log * see
 d1c6c5c

---
 youtube_dl/YoutubeDL.py | 21 +++++++++++++--------
 1 file changed, 13 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 212c04298..1b3ef94b4 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -102,6 +102,7 @@ from .utils import (
     YoutubeDLCookieProcessor,
     YoutubeDLHandler,
     YoutubeDLRedirectHandler,
+    ytdl_is_updateable,
 )
 from .cache import Cache
 from .extractor import get_info_extractor, gen_extractor_classes, _LAZY_LOADER
@@ -2373,9 +2374,11 @@ class YoutubeDL(object):
                 self.get_encoding()))
         write_string(encoding_str, encoding=None)
 
-        self._write_string('[debug] youtube-dl version ' + __version__ + '\n')
+        writeln_debug = lambda *s: self._write_string('[debug] %s\n' % (''.join(s), ))
+
+        writeln_debug('youtube-dl version ', __version__, (' (single file build)' if ytdl_is_updateable() else ''))
         if _LAZY_LOADER:
-            self._write_string('[debug] Lazy loading extractors enabled' + '\n')
+            writeln_debug('Lazy loading extractors enabled')
         try:
             sp = subprocess.Popen(
                 ['git', 'rev-parse', '--short', 'HEAD'],
@@ -2384,7 +2387,7 @@ class YoutubeDL(object):
             out, err = process_communicate_or_kill(sp)
             out = out.decode().strip()
             if re.match('[0-9a-f]+', out):
-                self._write_string('[debug] Git HEAD: ' + out + '\n')
+                writeln_debug('Git HEAD: ', out)
         except Exception:
             try:
                 sys.exc_clear()
@@ -2403,13 +2406,15 @@ class YoutubeDL(object):
             except OSError:  # We may not have access to the executable
                 return []
 
-        self._write_string('[debug] Python %s (%s %s) - %s (%s%s)\n' % (
+        libc = join_nonempty(*libc_ver(), delim=' ')
+        writeln_debug('Python %s (%s %s %s) - %s - %s%s' % (
             platform.python_version(),
             python_implementation(),
+            platform.machine(),
             platform.architecture()[0],
             platform_name(),
             OPENSSL_VERSION,
-            ', %s' % (join_nonempty(*libc_ver(), delim=' ') or '-'),
+            (' - %s' % (libc, )) if libc else ''
         ))
 
         exe_versions = FFmpegPostProcessor.get_versions(self)
@@ -2422,17 +2427,17 @@ class YoutubeDL(object):
         )
         if not exe_str:
             exe_str = 'none'
-        self._write_string('[debug] exe versions: %s\n' % exe_str)
+        writeln_debug('exe versions: %s' % (exe_str, ))
 
         proxy_map = {}
         for handler in self._opener.handlers:
             if hasattr(handler, 'proxies'):
                 proxy_map.update(handler.proxies)
-        self._write_string('[debug] Proxy map: ' + compat_str(proxy_map) + '\n')
+        writeln_debug('Proxy map: ', compat_str(proxy_map))
 
         if self.params.get('call_home', False):
             ipaddr = self.urlopen('https://yt-dl.org/ip').read().decode('utf-8')
-            self._write_string('[debug] Public IP address: %s\n' % ipaddr)
+            writeln_debug('Public IP address: %s' % (ipaddr, ))
             latest_version = self.urlopen(
                 'https://yt-dl.org/latest/version').read().decode('utf-8')
             if version_tuple(latest_version) > version_tuple(__version__):

From ee731f3d00064f446faa9ffb4c21ce4ca388bf5d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 23 May 2023 16:19:55 +0100
Subject: [PATCH 645/860] [ITV] Fix UA capitalisation in 384f632

---
 youtube_dl/extractor/itv.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index 7026139ea..c64af3be6 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -59,7 +59,7 @@ class ITVBaseIE(InfoExtractor):
 
     @staticmethod
     def _vanilla_ua_header():
-        return {'User-agent': 'Mozilla/5.0'}
+        return {'User-Agent': 'Mozilla/5.0'}
 
     def _download_webpage_handle(self, url, video_id, *args, **kwargs):
         # specialised to (a) use vanilla UA (b) detect geo-block
@@ -69,7 +69,7 @@ class ITVBaseIE(InfoExtractor):
                 'user_agent' not in params
                 and not any(re.match(r'(?i)user-agent\s*:', h)
                             for h in (params.get('headers') or []))
-                and 'User-agent' not in (kwargs.get('headers') or {})):
+                and 'User-Agent' not in (kwargs.get('headers') or {})):
 
             kwargs.setdefault('headers', {})
             kwargs['headers'] = self._vanilla_ua_header()

From 2389c7cbd30813435c50848a9b276bcfe2a810db Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 23 May 2023 17:11:22 +0100
Subject: [PATCH 646/860] [compat] Fix casefold import __all__ syntax in
 a19855f

---
 youtube_dl/casefold.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/casefold.py b/youtube_dl/casefold.py
index 748c2d491..ad9c66f8e 100644
--- a/youtube_dl/casefold.py
+++ b/youtube_dl/casefold.py
@@ -1663,5 +1663,5 @@ def casefold(s):
 
 
 __all__ = [
-    casefold
+    'casefold',
 ]

From b8a86dcf1aa837577178ae25357d8241ab4ba6c1 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 26 May 2023 20:25:25 +0100
Subject: [PATCH 647/860] [core] Revise 1f7c6f8 to help downstream merger
 (possibly)

---
 youtube_dl/YoutubeDL.py | 7 +++----
 1 file changed, 3 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1b3ef94b4..98b878fc1 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -2374,11 +2374,10 @@ class YoutubeDL(object):
                 self.get_encoding()))
         write_string(encoding_str, encoding=None)
 
-        writeln_debug = lambda *s: self._write_string('[debug] %s\n' % (''.join(s), ))
-
-        writeln_debug('youtube-dl version ', __version__, (' (single file build)' if ytdl_is_updateable() else ''))
+        self._write_string('[debug] youtube-dl version ' + __version__ + (' (single file build)\n' if ytdl_is_updateable() else '\n'))
         if _LAZY_LOADER:
-            writeln_debug('Lazy loading extractors enabled')
+            self._write_string('[debug] Lazy loading extractors enabled\n')
+        writeln_debug = lambda *s: self._write_string('[debug] %s\n' % (''.join(s), ))  # moved down for easier merge
         try:
             sp = subprocess.Popen(
                 ['git', 'rev-parse', '--short', 'HEAD'],

From a2534f7b888416e872d5afd1862eb3e30fc69fc7 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 11 Jun 2023 13:33:50 +0100
Subject: [PATCH 648/860] [jsinterp] Fix div bug breaking player 8c7583ff

Thx bashonly: https://github.com/ytdl-org/youtube-dl/issues/32292#issuecomment-1585639223
Fixes #32292
---
 test/test_jsinterp.py          | 49 ++++++++++++++++++++++++++++++++++
 test/test_youtube_signature.py |  4 +++
 youtube_dl/jsinterp.py         |  2 +-
 3 files changed, 54 insertions(+), 1 deletion(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 1cc148b15..ecd6ab3c9 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -33,6 +33,55 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('function x4(a){return 2*a+1;}')
         self.assertEqual(jsi.call_function('x4', 3), 7)
 
+    def test_add(self):
+        jsi = JSInterpreter('function f(){return 42 + 7;}')
+        self.assertEqual(jsi.call_function('f'), 49)
+        jsi = JSInterpreter('function f(){return 42 + undefined;}')
+        self.assertTrue(math.isnan(jsi.call_function('f')))
+        jsi = JSInterpreter('function f(){return 42 + null;}')
+        self.assertEqual(jsi.call_function('f'), 42)
+
+    def test_sub(self):
+        jsi = JSInterpreter('function f(){return 42 - 7;}')
+        self.assertEqual(jsi.call_function('f'), 35)
+        jsi = JSInterpreter('function f(){return 42 - undefined;}')
+        self.assertTrue(math.isnan(jsi.call_function('f')))
+        jsi = JSInterpreter('function f(){return 42 - null;}')
+        self.assertEqual(jsi.call_function('f'), 42)
+
+    def test_mul(self):
+        jsi = JSInterpreter('function f(){return 42 * 7;}')
+        self.assertEqual(jsi.call_function('f'), 294)
+        jsi = JSInterpreter('function f(){return 42 * undefined;}')
+        self.assertTrue(math.isnan(jsi.call_function('f')))
+        jsi = JSInterpreter('function f(){return 42 * null;}')
+        self.assertEqual(jsi.call_function('f'), 0)
+
+    def test_div(self):
+        jsi = JSInterpreter('function f(a, b){return a / b;}')
+        self.assertTrue(math.isnan(jsi.call_function('f', 0, 0)))
+        self.assertTrue(math.isnan(jsi.call_function('f', JS_Undefined, 1)))
+        self.assertTrue(math.isinf(jsi.call_function('f', 2, 0)))
+        self.assertEqual(jsi.call_function('f', 0, 3), 0)
+
+    def test_mod(self):
+        jsi = JSInterpreter('function f(){return 42 % 7;}')
+        self.assertEqual(jsi.call_function('f'), 0)
+        jsi = JSInterpreter('function f(){return 42 % 0;}')
+        self.assertTrue(math.isnan(jsi.call_function('f')))
+        jsi = JSInterpreter('function f(){return 42 % undefined;}')
+        self.assertTrue(math.isnan(jsi.call_function('f')))
+
+    def test_exp(self):
+        jsi = JSInterpreter('function f(){return 42 ** 2;}')
+        self.assertEqual(jsi.call_function('f'), 1764)
+        jsi = JSInterpreter('function f(){return 42 ** undefined;}')
+        self.assertTrue(math.isnan(jsi.call_function('f')))
+        jsi = JSInterpreter('function f(){return 42 ** null;}')
+        self.assertEqual(jsi.call_function('f'), 1)
+        jsi = JSInterpreter('function f(){return undefined ** 42;}')
+        self.assertTrue(math.isnan(jsi.call_function('f')))
+
     def test_empty_return(self):
         jsi = JSInterpreter('function f(){return; y()}')
         self.assertEqual(jsi.call_function('f'), None)
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index d41d708a0..e7bce9d68 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -151,6 +151,10 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/cfa9e7cb/player_ias.vflset/en_US/base.js',
         'qO0NiMtYQ7TeJnfFG2', 'k9cuJDHNS5O7kQ',
     ),
+    (
+        'https://www.youtube.com/s/player/8c7583ff/player_ias.vflset/en_US/base.js',
+        'E2AQVN6y_zM7uN9w8z', '9A2dbY5GDZrt9A',
+    ),
 ]
 
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index dc580943e..9d4a5bc57 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -82,7 +82,7 @@ def _js_arith_op(op):
 
 
 def _js_div(a, b):
-    if JS_Undefined in (a, b) or not (a and b):
+    if JS_Undefined in (a, b) or not (a or b):
         return _NaN
     return operator.truediv(a or 0, b) if b else float('inf')
 

From ff75c300f52321dc7322e28d1df153cf0ea65a6d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 17 Jun 2023 15:34:11 +0100
Subject: [PATCH 649/860] [jsinterp] Fix test for failed match in
 extract_object()

---
 youtube_dl/jsinterp.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 9d4a5bc57..c18c4fef1 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -985,9 +985,9 @@ class JSInterpreter(object):
                 \((?P<args>[^)]*)\)\s*
                 (?P<code>{.+})''' % {'name': re.escape(funcname)},
             self.code)
-        code, _ = self._separate_at_paren(func_m.group('code'))  # refine the match
         if func_m is None:
             raise self.Exception('Could not find JS function "{funcname}"'.format(**locals()))
+        code, _ = self._separate_at_paren(func_m.group('code'))  # refine the match
         return self.build_arglist(func_m.group('args')), code
 
     def extract_function(self, funcname):

From d6433cbb2c4440056a38846e35bb5a3efa9bcac2 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 17 Jun 2023 15:43:10 +0100
Subject: [PATCH 650/860] [jsinterp] Don't find unrelated objects

---
 youtube_dl/jsinterp.py | 14 +++++++-------
 1 file changed, 7 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index c18c4fef1..00f219440 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -941,15 +941,15 @@ class JSInterpreter(object):
         _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
         obj = {}
         obj_m = re.search(
-            r'''(?x)
-                (?<!this\.)%s\s*=\s*{\s*
-                    (?P<fields>(%s\s*:\s*function\s*\(.*?\)\s*{.*?}(?:,\s*)?)*)
-                }\s*;
-            ''' % (re.escape(objname), _FUNC_NAME_RE),
+            r'''(?xs)
+                (?:{0}\s*\.\s*{1}|{1}\s*=\s*\{{\s*
+                    (?P<fields>({2}\s*:\s*function\s*\(.*?\)\s*\{{.*?}}(?:,\s*)?)*)
+                }}\s*);
+            '''.format(_NAME_RE, re.escape(objname), _FUNC_NAME_RE),
             self.code)
-        if not obj_m:
+        fields = obj_m and obj_m.group('fields')
+        if fields is None:
             raise self.Exception('Could not find object ' + objname)
-        fields = obj_m.group('fields')
         # Currently, it only supports function definitions
         fields_m = re.finditer(
             r'''(?x)

From ae8ba2c31977b68b75221f80c488c0b12385269c Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 17 Jun 2023 15:36:39 +0100
Subject: [PATCH 651/860] [YouTube] Fix `KeyError QV` in signature extraction
 failed * temporarily force missing global definition into sig JS * improve
 test: thanks
 https://github.com/yt-dlp/yt-dlp/issues/7327#issuecomment-1595274615 *
 resolves #32314

---
 test/test_youtube_signature.py  | 7 ++++++-
 youtube_dl/extractor/youtube.py | 6 +++++-
 2 files changed, 11 insertions(+), 2 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index e7bce9d68..4ba586e53 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -63,6 +63,11 @@ _SIG_TESTS = [
         'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js',
         '312AA52209E3623129A412D56A40F11CB0AF14AE.3EE09501CB14E3BCDC3B2AE808BF3F1D14E7FBF12',
         '112AA5220913623229A412D56A40F11CB0AF14AE.3EE0950FCB14EEBCDC3B2AE808BF331D14E7FBF3',
+    ),
+    (
+        'https://www.youtube.com/s/player/6ed0d907/player_ias.vflset/en_US/base.js',
+        '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+        'AOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL2QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0',
     )
 ]
 
@@ -231,7 +236,7 @@ def n_sig(jscode, sig_input):
 
 
 make_sig_test = t_factory(
-    'signature', signature, re.compile(r'.*-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$'))
+    'signature', signature, re.compile(r'(?s).*(?:-|/player/)(?P<id>[a-zA-Z0-9_-]+)(?:/.+\.js|(?:/watch_as3|/html5player)?\.[a-z]+)$'))
 for test_spec in _SIG_TESTS:
     make_sig_test(*test_spec)
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0411c49f1..0bbce71a3 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1569,8 +1569,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
             jscode, 'Initial JS player signature function name', group='sig')
 
-        jsi = JSInterpreter(jscode)
+        # temporary (please) hack for player 6ed0d907 #32314
+        ah = 'var AH={LR:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c},QV:function(a){a.reverse()},pO:function(a,b){a.splice(0,b)}};'
+        jsi = JSInterpreter(ah + jscode)
+
         initial_function = jsi.extract_function(funcname)
+
         return lambda s: initial_function([s])
 
     def _decrypt_signature(self, s, video_id, player_url):

From 07af47960f3bb262ead02490ce65c8c45c01741e Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 18 Jun 2023 00:52:18 +0100
Subject: [PATCH 652/860] [YouTube] Improve fix for ae8ba2c Thx:
 https://github.com/yt-dlp/yt-dlp/commit/01aba25

---
 youtube_dl/extractor/youtube.py |  4 +---
 youtube_dl/jsinterp.py          | 21 ++++++++++++---------
 2 files changed, 13 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0bbce71a3..1855fca7f 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1569,9 +1569,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
             jscode, 'Initial JS player signature function name', group='sig')
 
-        # temporary (please) hack for player 6ed0d907 #32314
-        ah = 'var AH={LR:function(a,b){var c=a[0];a[0]=a[b%a.length];a[b%a.length]=c},QV:function(a){a.reverse()},pO:function(a,b){a.splice(0,b)}};'
-        jsi = JSInterpreter(ah + jscode)
+        jsi = JSInterpreter(jscode)
 
         initial_function = jsi.extract_function(funcname)
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 00f219440..1ba9c3d67 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -940,15 +940,18 @@ class JSInterpreter(object):
     def extract_object(self, objname):
         _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
         obj = {}
-        obj_m = re.search(
-            r'''(?xs)
-                (?:{0}\s*\.\s*{1}|{1}\s*=\s*\{{\s*
-                    (?P<fields>({2}\s*:\s*function\s*\(.*?\)\s*\{{.*?}}(?:,\s*)?)*)
-                }}\s*);
-            '''.format(_NAME_RE, re.escape(objname), _FUNC_NAME_RE),
-            self.code)
-        fields = obj_m and obj_m.group('fields')
-        if fields is None:
+        fields = None
+        for obj_m in re.finditer(
+                r'''(?xs)
+                    {0}\s*\.\s*{1}|{1}\s*=\s*\{{\s*
+                        (?P<fields>({2}\s*:\s*function\s*\(.*?\)\s*\{{.*?}}(?:,\s*)?)*)
+                    }}\s*;
+                '''.format(_NAME_RE, re.escape(objname), _FUNC_NAME_RE),
+                self.code):
+            fields = obj_m.group('fields')
+            if fields:
+                break
+        else:
             raise self.Exception('Could not find object ' + objname)
         # Currently, it only supports function definitions
         fields_m = re.finditer(

From 9112e668a5ea6376017718db9ff13b369d53ad7a Mon Sep 17 00:00:00 2001
From: pukkandan <pukkandan.ytdlp@gmail.com>
Date: Thu, 22 Jun 2023 13:23:31 +0530
Subject: [PATCH 653/860] [YouTube] Improve nsig function name extraction

Fixes player b7910ca8, using `,` vs `;`
See https://github.com/ytdl-org/youtube-dl/issues/32292#issuecomment-1602231170

Co-authored-by: dirkf
---
 test/test_youtube_signature.py  | 11 +++--------
 youtube_dl/extractor/youtube.py | 19 +++++++++++++------
 2 files changed, 16 insertions(+), 14 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 4ba586e53..5dcabaf95 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -63,11 +63,6 @@ _SIG_TESTS = [
         'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js',
         '312AA52209E3623129A412D56A40F11CB0AF14AE.3EE09501CB14E3BCDC3B2AE808BF3F1D14E7FBF12',
         '112AA5220913623229A412D56A40F11CB0AF14AE.3EE0950FCB14EEBCDC3B2AE808BF331D14E7FBF3',
-    ),
-    (
-        'https://www.youtube.com/s/player/6ed0d907/player_ias.vflset/en_US/base.js',
-        '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
-        'AOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL2QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0',
     )
 ]
 
@@ -157,8 +152,8 @@ _NSIG_TESTS = [
         'qO0NiMtYQ7TeJnfFG2', 'k9cuJDHNS5O7kQ',
     ),
     (
-        'https://www.youtube.com/s/player/8c7583ff/player_ias.vflset/en_US/base.js',
-        'E2AQVN6y_zM7uN9w8z', '9A2dbY5GDZrt9A',
+        'https://www.youtube.com/s/player/b7910ca8/player_ias.vflset/en_US/base.js',
+        '_hXMCwMt9qE310D', 'LoZMgkkofRMCZQ',
     ),
 ]
 
@@ -236,7 +231,7 @@ def n_sig(jscode, sig_input):
 
 
 make_sig_test = t_factory(
-    'signature', signature, re.compile(r'(?s).*(?:-|/player/)(?P<id>[a-zA-Z0-9_-]+)(?:/.+\.js|(?:/watch_as3|/html5player)?\.[a-z]+)$'))
+    'signature', signature, re.compile(r'.*-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$'))
 for test_spec in _SIG_TESTS:
     make_sig_test(*test_spec)
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1855fca7f..24e2efbd9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1623,15 +1623,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         nfunc, idx = re.match(target, nfunc_and_idx).group('nfunc', 'idx')
         if not idx:
             return nfunc
+
+        VAR_RE_TMPL = r'var\s+%s\s*=\s*(?P<name>\[(?P<alias>%s)\])[;,]'
+        note = 'Initial JS player n function {0} (%s[%s])' % (nfunc, idx)
+
+        def search_function_code(needle, group):
+            return self._search_regex(
+                VAR_RE_TMPL % (re.escape(nfunc), needle), jscode,
+                note.format(group), group=group)
+
         if int_or_none(idx) == 0:
-            real_nfunc = self._search_regex(
-                r'var %s\s*=\s*\[([a-zA-Z_$][\w$]*)\];' % (re.escape(nfunc), ), jscode,
-                'Initial JS player n function alias ({nfunc}[{idx}])'.format(**locals()))
+            real_nfunc = search_function_code(r'[a-zA-Z_$][\w$]*', group='alias')
             if real_nfunc:
                 return real_nfunc
-        return self._parse_json(self._search_regex(
-            r'var %s\s*=\s*(\[.+?\]);' % (re.escape(nfunc), ), jscode,
-            'Initial JS player n function name ({nfunc}[{idx}])'.format(**locals())), nfunc, transform_source=js_to_json)[int(idx)]
+        return self._parse_json(
+            search_function_code('.+?', group='name'),
+            nfunc, transform_source=js_to_json)[int(idx)]
 
     def _extract_n_function(self, video_id, player_url):
         player_id = self._extract_player_info(player_url)

From ebdc82c58684b4e202fabc046f9a40fc73cccde5 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 22 Jun 2023 17:24:48 +0100
Subject: [PATCH 654/860] [workflows/ci.yml] Replace actions/setup-python for
 legacy Pythons

Thanks MatteoH2O1999: https://github.com/MatteoH2O1999/setup-python
---
 .github/workflows/ci.yml | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 51abdce1d..9be4eaa89 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -38,10 +38,12 @@ jobs:
     steps:
     - uses: actions/checkout@v3
     - name: Set up supported Python ${{ matrix.python-version }}
-      uses: actions/setup-python@v4
-      if: ${{ matrix.python-impl == 'cpython' && ! contains(fromJSON('["3.3", "3.4"]'), matrix.python-version) }}
+      # wrap broken actions/setup-python@v4
+      uses: ytdl-org/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
+        cache-build: true
+        allow-build: info
     - name: Set up Java 8
       if: ${{ matrix.python-impl == 'jython' }}
       uses: actions/setup-java@v2

From fa7f0effbe4e14fcf70e1dc4496371c9862b64b9 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 22 Jun 2023 23:10:04 +0100
Subject: [PATCH 655/860] [YouTube] Avoid crash in author extraction

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 24e2efbd9..9c419c002 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -448,7 +448,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             extract_attributes(self._search_regex(
                 r'''(?s)(<link\b[^>]+\bitemprop\s*=\s*("|')%s\2[^>]*>)'''
                 % re.escape(var_name),
-                get_element_by_attribute('itemprop', 'author', webpage) or '',
+                get_element_by_attribute('itemprop', 'author', webpage or '') or '',
                 'author link', default='')),
             paths[var_name][0])
 

From 58fc5bde47215d9e7c60647dd21202a254b3b066 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 23 Jun 2023 00:15:06 +0100
Subject: [PATCH 656/860] [workflows/ci.yml] Restore test support for Py 3.3,
 3.4, and add 2.6

---
 .github/workflows/ci.yml | 4 +---
 1 file changed, 1 insertion(+), 3 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9be4eaa89..4008cc190 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -8,9 +8,7 @@ jobs:
       fail-fast: true
       matrix:
         os: [ubuntu-20.04]
-        # TODO: python 2.6
-        # TODO: restore support for 3.3, 3.4
-        python-version: [2.7, 3.5, 3.6, 3.7, 3.8, 3.9, pypy-2.7, pypy-3.6, pypy-3.7]
+        python-version: [2.6, 2.7, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, pypy-2.7, pypy-3.6, pypy-3.7]
         python-impl: [cpython]
         ytdl-test-set: [core, download]
         run-tests-ext: [sh]

From 2500300c2a5986ace34390aa473a8bd51f83622c Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 29 Jun 2023 15:27:12 +0100
Subject: [PATCH 657/860] [workflows/ci.yml] Restore test support for Py 3.2

---
 .github/workflows/ci.yml           | 319 +++++++++++++++++++++++++++--
 devscripts/make_lazy_extractors.py |   4 +
 test/test_execution.py             |   8 +-
 test/test_unicode_literals.py      |   1 +
 youtube_dl/__init__.py             |   8 +-
 youtube_dl/compat.py               |  18 +-
 6 files changed, 328 insertions(+), 30 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 4008cc190..8d8e654fb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,73 +1,349 @@
 name: CI
-on: [push, pull_request]
+
+env:
+  # add 3.10+ after patching nose (https://github.com/nose-devs/nose/issues/1099)
+  # or switching to fork of https://github.com/mdmintz/pynose
+  all-cpython-versions: 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9
+  main-cpython-versions: 2.7, 3.2, 3.5, 3.9
+  pypy-versions: pypy-2.7, pypy-3.6, pypy-3.7
+  cpython-versions: all
+  # test-set: both
+  test-set: core
+
+on:
+  push:
+  pull_request:
+  workflow_dispatch:
+    inputs:
+      cpython-versions:
+        type: choice
+        description: CPython versions (main = 2.7, 3.2, 3.5, 3.9)
+        options:
+          - all
+          - main
+        required: true
+        default: main
+      test-set:
+        type: choice
+        description: core, download
+        options:
+          - both
+          - core
+          - download
+        required: true
+        default: core
+
+permissions:
+  contents: read
+
 jobs:
+  select:
+    name: Select tests from inputs
+    runs-on: ubuntu-latest
+    outputs:
+      cpython-versions: ${{ steps.run.outputs.cpython-versions }}
+      test-set: ${{ steps.run.outputs.test-set }}
+      own-pip-versions: ${{ steps.run.outputs.own-pip-versions }}
+    steps:
+    - id: run
+      run: |
+        # Make a JSON Array from comma/space-separated string (no extra escaping)
+        json_list() { \
+          ret=""; IFS="${IFS},"; set -- $*; \
+          for a in "$@"; do \
+            ret=$(printf '%s"%s"' "${ret}${ret:+, }" "$a"); \
+          done; \
+          printf '[%s]' "$ret"; }
+        tests="${{ inputs.test-set || env.test-set }}"
+        [ $tests = both ] && tests="core download"
+        printf 'test-set=%s\n' "$(json_list $tests)" >> "$GITHUB_OUTPUT"
+        versions="${{ inputs.cpython-versions || env.cpython-versions }}"
+        if [ "$versions" = all ]; then \
+          versions="${{ env.all-cpython-versions }}"; else \
+          versions="${{ env.main-cpython-versions }}"; \
+        fi
+        printf 'cpython-versions=%s\n' \
+          "$(json_list ${versions}${versions:+, }${{ env.pypy-versions }})" >> "$GITHUB_OUTPUT"
+        # versions with a special get-pip.py in a per-version subdirectory
+        printf 'own-pip-versions=%s\n' \
+          "$(json_list 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6)" >> "$GITHUB_OUTPUT"
+
   tests:
-    name: Tests
+    name: Run tests
+    needs: select
+    permissions:
+      contents: read
+      packages: write
     runs-on: ${{ matrix.os }}
     strategy:
       fail-fast: true
       matrix:
         os: [ubuntu-20.04]
-        python-version: [2.6, 2.7, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, pypy-2.7, pypy-3.6, pypy-3.7]
+        # outside steps, use github.env...., not env....
+        python-version: ${{ fromJSON(needs.select.outputs.cpython-versions) }}
         python-impl: [cpython]
-        ytdl-test-set: [core, download]
+        ytdl-test-set: ${{ fromJSON(needs.select.outputs.test-set) }}
         run-tests-ext: [sh]
         include:
-        # python 3.2 is only available on windows via setup-python
         - os: windows-2019
           python-version: 3.2
           python-impl: cpython
-          ytdl-test-set: core
+          ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }}
           run-tests-ext: bat
         - os: windows-2019
           python-version: 3.2
           python-impl: cpython
-          ytdl-test-set: download
+          ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download'  || 'nodownload' }}
           run-tests-ext: bat
         # jython
         - os: ubuntu-20.04
           python-impl: jython
-          ytdl-test-set: core
+          ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }}
           run-tests-ext: sh
         - os: ubuntu-20.04
           python-impl: jython
-          ytdl-test-set: download
+          ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download'  || 'nodownload' }}
           run-tests-ext: sh
     steps:
-    - uses: actions/checkout@v3
+    - name: Checkout
+      uses: actions/checkout@v3
+    #-------- Python 3 -----
     - name: Set up supported Python ${{ matrix.python-version }}
+      id: setup-python
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version != '2.6' && matrix.python-version != '2.7'}}
       # wrap broken actions/setup-python@v4
       uses: ytdl-org/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}
         cache-build: true
         allow-build: info
+    - name: Locate supported Python ${{ matrix.python-version }}
+      if: ${{ env.pythonLocation }}
+      shell: bash
+      run: |
+        echo "PYTHONHOME=${pythonLocation}" >> "$GITHUB_ENV"
+        export expected="${{ steps.setup-python.outputs.python-path }}"
+        dirname() { printf '%s\n' \
+            'import os, sys' \
+            'print(os.path.dirname(sys.argv[1]))' \
+            | ${expected} - "$1"; }
+        expd="$(dirname "$expected")"
+        export python="$(command -v python)"
+        [ "$expd" = "$(dirname "$python")" ] || echo "PATH=$expd:${PATH}" >> "$GITHUB_ENV"
+        [ -x "$python" ] || printf '%s\n' \
+            'import os' \
+            'exp = os.environ["expected"]' \
+            'python = os.environ["python"]' \
+            'exps = os.path.split(exp)' \
+            'if python and (os.path.dirname(python) == exp[0]):' \
+            '    exit(0)' \
+            'exps[1] = "python" + os.path.splitext(exps[1])[1]' \
+            'python = os.path.join(*exps)' \
+            'try:' \
+            '    os.symlink(exp, python)' \
+            'except AttributeError:' \
+            '    os.rename(exp, python)' \
+            | ${expected} -
+        printf '%s\n' \
+            'import sys' \
+            'print(sys.path)' \
+            | ${expected} -
+    #-------- Python 2.7 --
+    - name: Set up Python 2.7
+      if: ${{ matrix.python-version == '2.7' }}
+      # install 2.7
+      run: |
+        sudo apt-get install -y python2 python-is-python2
+        echo "PYTHONHOME=/usr" >> "$GITHUB_ENV"
+    #-------- Python 2.6 --
+    - name: Set up Python 2.6 environment
+      if: ${{ matrix.python-version == '2.6' }}
+      run: |
+        openssl_name=openssl-1.0.2u
+        echo "openssl_name=${openssl_name}" >> "$GITHUB_ENV"
+        openssl_dir=$HOME/.local/opt/$openssl_name
+        echo "openssl_dir=${openssl_dir}" >> "$GITHUB_ENV"
+        PYENV_ROOT=$HOME/.local/share/pyenv
+        echo "PYENV_ROOT=${PYENV_ROOT}" >> "$GITHUB_ENV"
+        sudo apt-get install -y openssl ca-certificates
+    - name: Cache Python 2.6
+      id: cache26
+      if: ${{ matrix.python-version == '2.6' }}
+      uses: actions/cache@v3
+      with:
+        key: python-2.6.9
+        path: |
+          ${{ env.openssl_dir }}
+          ${{ env.PYENV_ROOT }}
+    - name: Build and set up Python 2.6
+      if: ${{ matrix.python-version == '2.6' && ! steps.cache26.outputs.cache-hit }}
+      # dl and build locally
+      run: |
+        # Install build environment
+        sudo apt-get install -y build-essential llvm libssl-dev tk-dev  \
+                      libncursesw5-dev libreadline-dev libsqlite3-dev   \
+                      libffi-dev xz-utils zlib1g-dev libbz2-dev liblzma-dev
+        # Download and install OpenSSL 1.0.2, back in time
+        openssl_name=${{ env.openssl_name }}
+        openssl_targz=${openssl_name}.tar.gz
+        openssl_dir=${{ env.openssl_dir }}
+        openssl_inc=$openssl_dir/include
+        openssl_lib=$openssl_dir/lib
+        openssl_ssl=$openssl_dir/ssl
+        curl -L "https://www.openssl.org/source/$openssl_targz" -o $openssl_targz
+        tar -xf $openssl_targz
+        ( cd $openssl_name; \
+          ./config --prefix=$openssl_dir --openssldir=${openssl_dir}/ssl \
+            --libdir=lib -Wl,-rpath=${openssl_dir}/lib shared zlib-dynamic && \
+          make && \
+          make install )
+        rm -rf $openssl_name
+        rmdir $openssl_ssl/certs && ln -s /etc/ssl/certs $openssl_ssl/certs
+
+        # Download PyEnv from its GitHub repository.
+        export PYENV_ROOT=${{ env.PYENV_ROOT }}
+        export PATH=$PYENV_ROOT/bin:$PATH
+        git clone https://github.com/pyenv/pyenv.git $PYENV_ROOT
+        eval "$(pyenv init --path)"
+
+        # Prevent pyenv build trying (and failing) to update pip
+        export GET_PIP=get-pip-2.6.py
+        echo 'import sys; sys.exit(0)' > ${GET_PIP}
+        GET_PIP=$(realpath $GET_PIP)
+
+        # Build and install Python
+        export CFLAGS="-I$openssl_inc"
+        export LDFLAGS="-L$openssl_lib"
+        export LD_LIBRARY_PATH="$openssl_lib"
+        pyenv install 2.6.9
+        echo "PYTHONHOME=${PYENV_ROOT}" >> "$GITHUB_ENV"
+        echo "PATH=$PYENV_ROOT/bin:$PATH" >> "$GITHUB_ENV"
+    - name: Set up cached Python 2.6
+      if: ${{ steps.cache26.outputs.cache-hit }}
+      run: |
+        export PYENV_ROOT
+        export PATH=$PYENV_ROOT/bin:$PATH
+        eval "$(pyenv init --path)"
+        pyenv local 2.6.9
+        echo "PYTHONHOME=${PYENV_ROOT}" >> "$GITHUB_ENV"
+        echo "PATH=$PYENV_ROOT/bin:$PATH" >> "$GITHUB_ENV"
+    #-------- Jython ------
     - name: Set up Java 8
       if: ${{ matrix.python-impl == 'jython' }}
       uses: actions/setup-java@v2
       with:
         java-version: 8
         distribution: 'zulu'
-    - name: Install Jython
+    - name: Setup Jython environment
       if: ${{ matrix.python-impl == 'jython' }}
       run: |
-        wget https://repo1.maven.org/maven2/org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar -O jython-installer.jar
-        java -jar jython-installer.jar -s -d "$HOME/jython"
-        echo "$HOME/jython/bin" >> $GITHUB_PATH
-    - name: Install nose
-      if: ${{ matrix.python-impl != 'jython' }}
-      run: pip install nose
+        echo "JYTHON_ROOT=${HOME}/jython" >> "$GITHUB_ENV"
+    - name: Cache Jython
+      id: cachejy
+      if: ${{ matrix.python-impl == 'jython' }}
+      uses: actions/cache@v3
+      with:
+        # 2.7.3 now available, may solve SNI issue
+        key: jython-2.7.1
+        path: |
+          ${{ env.JYTHON_ROOT }}
+    - name: Install Jython
+      if: ${{ matrix.python-impl == 'jython' && ! steps.cachejy.outputs.cache-hit }}
+      run: |
+        JYTHON_ROOT="${{ env.JYTHON_ROOT }}"
+        curl -L "https://repo1.maven.org/maven2/org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar" -o jython-installer.jar
+        java -jar jython-installer.jar -s -d "${JYTHON_ROOT}"
+        echo "${JYTHON_ROOT}/bin" >> $GITHUB_PATH
+    - name: Set up cached Jython
+      if: ${{ steps.cachejy.outputs.cache-hit }}
+      run: |
+        JYTHON_ROOT="${{ env.JYTHON_ROOT }}"
+        echo "${JYTHON_ROOT}/bin" >> $GITHUB_PATH
+    #-------- pip ---------
+    - name: Set up supported Python ${{ matrix.python-version }} pip
+      if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.6' || matrix.python-version == '2.7' }}
+      # This step may run in either Linux or Windows
+      shell: bash
+      run: |
+        echo "$PATH"
+        echo "$PYTHONHOME"
+        # curl is available on both Windows and Linux, -L follows redirects, -O gets name
+        python -m ensurepip || python -m pip --version || { \
+          get_pip="${{ contains(needs.select.outputs.own-pip-versions, matrix.python-version) && format('{0}/', matrix.python-version) || '' }}"; \
+          curl -L -O "https://bootstrap.pypa.io/pip/${get_pip}get-pip.py"; \
+          python get-pip.py; }
+    - name: Set up other Python ${{ matrix.python-version }} pip
+      if: ${{ matrix.python-version == '3.2' && steps.setup-python.outputs.python-path }}
+      shell: bash
+      run: |
+        # https://files.pythonhosted.org/packages/8a/e9/8468cd68b582b06ef554be0b96b59f59779627131aad48f8a5bce4b13450/wheel-0.29.0-py2.py3-none-any.whl
+        # https://files.pythonhosted.org/packages/06/4b/86a670fd21f7849adb092e40883c48dcd0d66b8a878fc8d63b7f0ea04213/setuptools-29.0.1-py2.py3-none-any.whl
+        python -m pip --version || { \
+          curl -L -O "https://bootstrap.pypa.io/pip/3.2/get-pip.py"; \
+          curl -L -O "https://files.pythonhosted.org/packages/b2/d0/cd115fe345dd6f07ec1c780020a7dfe74966fceeb171e0f20d1d4905b0b7/pip-7.1.2-py2.py3-none-any.whl"; \
+          python -v get-pip.py --no-setuptools --no-wheel pip-7.1.2-py2.py3-none-any.whl; }
+
+    #-------- nose --------
+    - name: Install nose for Python ${{ matrix.python-version }}
+      if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.6' || matrix.python-version == '2.7' }}
+      shell: bash
+      run: |
+        echo "$PATH"
+        echo "$PYTHONHOME"
+        python --version
+        python -m pip --version
+        python -m pip nose --version || python -m pip install nose
+    - name: Install nose for other Python ${{ matrix.python-version }}
+      if: ${{ matrix.python-version == '3.2' && steps.setup-python.outputs.python-path }}
+      shell: bash
+      run: |
+        python -m pip nose --version || { \
+          curl -L -O "https://files.pythonhosted.org/packages/15/d8/dd071918c040f50fa1cf80da16423af51ff8ce4a0f2399b7bf8de45ac3d9/nose-1.3.7-py3-none-any.whl"; \
+          python --version; \
+          printf '%s\n' \
+            'import sys' \
+            'print(sys.path)' \
+            | python -; \
+          python -m pip --version; \
+          python -m pip install nose-1.3.7-py3-none-any.whl; }
     - name: Install nose (Jython)
       if: ${{ matrix.python-impl == 'jython' }}
-      # Working around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb)
+      # Work around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb)
       run: |
-        wget https://files.pythonhosted.org/packages/99/4f/13fb671119e65c4dce97c60e67d3fd9e6f7f809f2b307e2611f4701205cb/nose-1.3.7-py2-none-any.whl
-        pip install nose-1.3.7-py2-none-any.whl
+        pip nose --version || { \
+          curl -L -O "https://files.pythonhosted.org/packages/99/4f/13fb671119e65c4dce97c60e67d3fd9e6f7f809f2b307e2611f4701205cb/nose-1.3.7-py2-none-any.whl"; \
+          pip --version; \
+          pip install nose-1.3.7-py2-none-any.whl; }
+    - name: Set up nosetest test
+      if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }}
+      shell: bash
+      run: |
+        # define a test to validate the Python version used by nosetests
+        printf '%s\n' \
+          'from __future__ import unicode_literals' \
+          'import sys, os, platform, unittest' \
+          'class TestPython(unittest.TestCase):' \
+          '    def setUp(self):' \
+          '        self.ver = os.environ["PYTHON_VER"].split("-")' \
+          '    def test_python_ver(self):' \
+          '        self.assertEqual(sys.version[:3], self.ver[-1])' \
+          '        self.assertTrue(sys.version.startswith(self.ver[-1]))' \
+          '        self.assertIn(self.ver[0], sys.version.lower())' \
+          '    def test_python_impl(self):' \
+          '        self.assertIn(platform.python_implementation().lower(), (os.environ["PYTHON_IMPL"], self.ver[0]))' \
+          > test/test_python.py
+    #-------- TESTS -------
     - name: Run tests
+      if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }}
       continue-on-error: ${{ matrix.ytdl-test-set == 'download' || matrix.python-impl == 'jython' }}
       env:
         YTDL_TEST_SET: ${{ matrix.ytdl-test-set }}
-      run: ./devscripts/run_tests.${{ matrix.run-tests-ext }}
+        PYTHON_VER: ${{ matrix.python-version }}
+        PYTHON_IMPL: ${{ matrix.python-impl }}
+
+      run: |
+        ./devscripts/run_tests.${{ matrix.run-tests-ext }}
+
   flake8:
     name: Linter
     runs-on: ubuntu-latest
@@ -81,3 +357,4 @@ jobs:
       run: pip install flake8
     - name: Run flake8
       run: flake8 .
+
diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py
index edc19183d..4bddca047 100644
--- a/devscripts/make_lazy_extractors.py
+++ b/devscripts/make_lazy_extractors.py
@@ -6,6 +6,10 @@ import os
 from os.path import dirname as dirn
 import sys
 
+from youtube_dl.compat import compat_register_utf8
+
+compat_register_utf8()
+
 print('WARNING: Lazy loading extractors is an experimental feature that may not always work', file=sys.stderr)
 
 sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
diff --git a/test/test_execution.py b/test/test_execution.py
index 704e14612..1dee53a0f 100644
--- a/test/test_execution.py
+++ b/test/test_execution.py
@@ -10,10 +10,13 @@ import os
 import subprocess
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+from youtube_dl.compat import compat_register_utf8
+
 from youtube_dl.utils import encodeArgument
 
 rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
+compat_register_utf8()
 
 try:
     _DEV_NULL = subprocess.DEVNULL
@@ -25,13 +28,14 @@ class TestExecution(unittest.TestCase):
     def test_import(self):
         subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
 
+    @unittest.skipIf(sys.version_info < (2, 7), 'Python 2.6 doesn\'t support package execution')
     def test_module_exec(self):
-        if sys.version_info >= (2, 7):  # Python 2.6 doesn't support package execution
-            subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+        subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)
 
     def test_main_exec(self):
         subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL)
 
+    @unittest.skipIf(sys.version_info < (2, 7), 'Python 2.6 doesn\'t support package execution')
     def test_cmdline_umlauts(self):
         p = subprocess.Popen(
             [sys.executable, 'youtube_dl/__main__.py', encodeArgument('ä'), '--version'],
diff --git a/test/test_unicode_literals.py b/test/test_unicode_literals.py
index 6c1b7ec91..c7c2252f5 100644
--- a/test/test_unicode_literals.py
+++ b/test/test_unicode_literals.py
@@ -15,6 +15,7 @@ IGNORED_FILES = [
     'setup.py',  # http://bugs.python.org/issue13943
     'conf.py',
     'buildserver.py',
+    'get-pip.py',
 ]
 
 IGNORED_DIRS = [
diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index e1bd67919..cc8285eba 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -5,7 +5,6 @@ from __future__ import unicode_literals
 
 __license__ = 'Public Domain'
 
-import codecs
 import io
 import os
 import random
@@ -17,6 +16,7 @@ from .options import (
 )
 from .compat import (
     compat_getpass,
+    compat_register_utf8,
     compat_shlex_split,
     workaround_optparse_bug9161,
 )
@@ -46,10 +46,8 @@ from .YoutubeDL import YoutubeDL
 
 
 def _real_main(argv=None):
-    # Compatibility fixes for Windows
-    if sys.platform == 'win32':
-        # https://github.com/ytdl-org/youtube-dl/issues/820
-        codecs.register(lambda name: codecs.lookup('utf-8') if name == 'cp65001' else None)
+    # Compatibility fix for Windows
+    compat_register_utf8()
 
     workaround_optparse_bug9161()
 
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index fe62caf80..0f4d3756f 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -31,13 +31,17 @@ try:
     compat_str, compat_basestring, compat_chr = (
         unicode, basestring, unichr
     )
-    from .casefold import casefold as compat_casefold
-
 except NameError:
     compat_str, compat_basestring, compat_chr = (
         str, str, chr
     )
+
+# casefold
+try:
+    compat_str.casefold
     compat_casefold = lambda s: s.casefold()
+except AttributeError:
+    from .casefold import casefold as compat_casefold
 
 try:
     import collections.abc as compat_collections_abc
@@ -3137,6 +3141,15 @@ else:
     compat_open = open
 
 
+# compat_register_utf8
+def compat_register_utf8():
+    if sys.platform == 'win32':
+        # https://github.com/ytdl-org/youtube-dl/issues/820
+        from codecs import register, lookup
+        register(
+            lambda name: lookup('utf-8') if name == 'cp65001' else None)
+
+
 legacy = [
     'compat_HTMLParseError',
     'compat_HTMLParser',
@@ -3203,6 +3216,7 @@ __all__ = [
     'compat_print',
     'compat_re_Match',
     'compat_re_Pattern',
+    'compat_register_utf8',
     'compat_setenv',
     'compat_shlex_quote',
     'compat_shlex_split',

From b08a58090635777f1001d5cde2cd141a5565177c Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 30 Jun 2023 03:52:39 +0100
Subject: [PATCH 658/860] [workflows/ci.yml] Fix test support for Py 2.6

---
 .github/workflows/ci.yml           | 115 ++++++++++++++++++-----------
 devscripts/make_lazy_extractors.py |   8 +-
 test/test_execution.py             |  16 ++--
 3 files changed, 83 insertions(+), 56 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 8d8e654fb..ce878c1b1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -6,9 +6,8 @@ env:
   all-cpython-versions: 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9
   main-cpython-versions: 2.7, 3.2, 3.5, 3.9
   pypy-versions: pypy-2.7, pypy-3.6, pypy-3.7
-  cpython-versions: all
-  # test-set: both
-  test-set: core
+  cpython-versions: main
+  test-set: both
 
 on:
   push:
@@ -75,6 +74,10 @@ jobs:
       contents: read
       packages: write
     runs-on: ${{ matrix.os }}
+    env:
+      PIP: python -m pip
+      PIP_DISABLE_PIP_VERSION_CHECK: true
+      PIP_NO_PYTHON_VERSION_WARNING: true
     strategy:
       fail-fast: true
       matrix:
@@ -152,12 +155,14 @@ jobs:
     - name: Set up Python 2.7
       if: ${{ matrix.python-version == '2.7' }}
       # install 2.7
+      shell: bash
       run: |
         sudo apt-get install -y python2 python-is-python2
         echo "PYTHONHOME=/usr" >> "$GITHUB_ENV"
     #-------- Python 2.6 --
     - name: Set up Python 2.6 environment
       if: ${{ matrix.python-version == '2.6' }}
+      shell: bash
       run: |
         openssl_name=openssl-1.0.2u
         echo "openssl_name=${openssl_name}" >> "$GITHUB_ENV"
@@ -178,6 +183,7 @@ jobs:
     - name: Build and set up Python 2.6
       if: ${{ matrix.python-version == '2.6' && ! steps.cache26.outputs.cache-hit }}
       # dl and build locally
+      shell: bash
       run: |
         # Install build environment
         sudo apt-get install -y build-essential llvm libssl-dev tk-dev  \
@@ -203,8 +209,7 @@ jobs:
         # Download PyEnv from its GitHub repository.
         export PYENV_ROOT=${{ env.PYENV_ROOT }}
         export PATH=$PYENV_ROOT/bin:$PATH
-        git clone https://github.com/pyenv/pyenv.git $PYENV_ROOT
-        eval "$(pyenv init --path)"
+        git clone "https://github.com/pyenv/pyenv.git" "$PYENV_ROOT"
 
         # Prevent pyenv build trying (and failing) to update pip
         export GET_PIP=get-pip-2.6.py
@@ -216,17 +221,14 @@ jobs:
         export LDFLAGS="-L$openssl_lib"
         export LD_LIBRARY_PATH="$openssl_lib"
         pyenv install 2.6.9
-        echo "PYTHONHOME=${PYENV_ROOT}" >> "$GITHUB_ENV"
-        echo "PATH=$PYENV_ROOT/bin:$PATH" >> "$GITHUB_ENV"
-    - name: Set up cached Python 2.6
-      if: ${{ steps.cache26.outputs.cache-hit }}
+    - name: Locate Python 2.6
+      if: ${{ matrix.python-version == '2.6' }}
+      shell: bash
       run: |
-        export PYENV_ROOT
-        export PATH=$PYENV_ROOT/bin:$PATH
-        eval "$(pyenv init --path)"
-        pyenv local 2.6.9
-        echo "PYTHONHOME=${PYENV_ROOT}" >> "$GITHUB_ENV"
-        echo "PATH=$PYENV_ROOT/bin:$PATH" >> "$GITHUB_ENV"
+        PYTHONHOME="${{ env.PYENV_ROOT }}/versions/2.6.9"
+        echo "PYTHONHOME=$PYTHONHOME" >> "$GITHUB_ENV"
+        echo "PATH=${PYTHONHOME}/bin:$PATH" >> "$GITHUB_ENV"
+        echo "LD_LIBRARY_PATH=${{ env.openssl_dir }}/lib${LD_LIBRARY_PATH:+:}${LD_LIBRARY_PATH}" >> "$GITHUB_ENV"
     #-------- Jython ------
     - name: Set up Java 8
       if: ${{ matrix.python-impl == 'jython' }}
@@ -236,8 +238,10 @@ jobs:
         distribution: 'zulu'
     - name: Setup Jython environment
       if: ${{ matrix.python-impl == 'jython' }}
+      shell: bash
       run: |
         echo "JYTHON_ROOT=${HOME}/jython" >> "$GITHUB_ENV"
+        echo "PIP=pip" >> "$GITHUB_ENV"
     - name: Cache Jython
       id: cachejy
       if: ${{ matrix.python-impl == 'jython' }}
@@ -249,19 +253,21 @@ jobs:
           ${{ env.JYTHON_ROOT }}
     - name: Install Jython
       if: ${{ matrix.python-impl == 'jython' && ! steps.cachejy.outputs.cache-hit }}
+      shell: bash
       run: |
         JYTHON_ROOT="${{ env.JYTHON_ROOT }}"
         curl -L "https://repo1.maven.org/maven2/org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar" -o jython-installer.jar
         java -jar jython-installer.jar -s -d "${JYTHON_ROOT}"
-        echo "${JYTHON_ROOT}/bin" >> $GITHUB_PATH
+        echo "${JYTHON_ROOT}/bin" >> "$GITHUB_PATH"
     - name: Set up cached Jython
       if: ${{ steps.cachejy.outputs.cache-hit }}
+      shell: bash
       run: |
         JYTHON_ROOT="${{ env.JYTHON_ROOT }}"
         echo "${JYTHON_ROOT}/bin" >> $GITHUB_PATH
     #-------- pip ---------
     - name: Set up supported Python ${{ matrix.python-version }} pip
-      if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.6' || matrix.python-version == '2.7' }}
+      if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.7' }}
       # This step may run in either Linux or Windows
       shell: bash
       run: |
@@ -272,48 +278,66 @@ jobs:
           get_pip="${{ contains(needs.select.outputs.own-pip-versions, matrix.python-version) && format('{0}/', matrix.python-version) || '' }}"; \
           curl -L -O "https://bootstrap.pypa.io/pip/${get_pip}get-pip.py"; \
           python get-pip.py; }
+    - name: Set up Python 2.6 pip
+      if: ${{ matrix.python-version == '2.6' }}
+      shell: bash
+      run: |
+        python -m pip --version || { \
+          curl -L -O "https://bootstrap.pypa.io/pip/2.6/get-pip.py"; \
+          curl -L -O "https://files.pythonhosted.org/packages/ac/95/a05b56bb975efa78d3557efa36acaf9cf5d2fd0ee0062060493687432e03/pip-9.0.3-py2.py3-none-any.whl"; \
+          python get-pip.py --no-setuptools --no-wheel pip-9.0.3-py2.py3-none-any.whl; }
+        # work-around to invoke pip module on 2.6: https://bugs.python.org/issue2751
+        echo "PIP=python -m pip.__main__" >> "$GITHUB_ENV"
     - name: Set up other Python ${{ matrix.python-version }} pip
       if: ${{ matrix.python-version == '3.2' && steps.setup-python.outputs.python-path }}
       shell: bash
       run: |
-        # https://files.pythonhosted.org/packages/8a/e9/8468cd68b582b06ef554be0b96b59f59779627131aad48f8a5bce4b13450/wheel-0.29.0-py2.py3-none-any.whl
-        # https://files.pythonhosted.org/packages/06/4b/86a670fd21f7849adb092e40883c48dcd0d66b8a878fc8d63b7f0ea04213/setuptools-29.0.1-py2.py3-none-any.whl
         python -m pip --version || { \
           curl -L -O "https://bootstrap.pypa.io/pip/3.2/get-pip.py"; \
           curl -L -O "https://files.pythonhosted.org/packages/b2/d0/cd115fe345dd6f07ec1c780020a7dfe74966fceeb171e0f20d1d4905b0b7/pip-7.1.2-py2.py3-none-any.whl"; \
-          python -v get-pip.py --no-setuptools --no-wheel pip-7.1.2-py2.py3-none-any.whl; }
-
+          python get-pip.py --no-setuptools --no-wheel pip-7.1.2-py2.py3-none-any.whl; }
+    #-------- unittest ----
+    - name: Upgrade Unittest for Python 2.6
+      if: ${{ matrix.python-version == '2.6' }}
+      shell: bash
+      run: |
+        # see pip for Jython
+        $PIP -qq show unittest2 || { \
+          for u in "65/26/32b8464df2a97e6dd1b656ed26b2c194606c16fe163c695a992b36c11cdf/six-1.13.0-py2.py3-none-any.whl" \
+              "f2/94/3af39d34be01a24a6e65433d19e107099374224905f1e0cc6bbe1fd22a2f/argparse-1.4.0-py2.py3-none-any.whl" \
+              "c7/a3/c5da2a44c85bfbb6eebcfc1dde24933f8704441b98fdde6528f4831757a6/linecache2-1.0.0-py2.py3-none-any.whl" \
+              "17/0a/6ac05a3723017a967193456a2efa0aa9ac4b51456891af1e2353bb9de21e/traceback2-1.4.0-py2.py3-none-any.whl" \
+              "72/20/7f0f433060a962200b7272b8c12ba90ef5b903e218174301d0abfd523813/unittest2-1.1.0-py2.py3-none-any.whl"; do \
+            curl -L -O "https://files.pythonhosted.org/packages/${u}"; \
+            $PIP install ${u##*/}; \
+          done; }
+        # make tests use unittest2
+        for test in ./test/test_*.py; do
+          sed -r -i -e '/^import unittest$/s/test/test2 as unittest/' "$test"
+        done
     #-------- nose --------
     - name: Install nose for Python ${{ matrix.python-version }}
-      if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.6' || matrix.python-version == '2.7' }}
+      if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.7' }}
       shell: bash
       run: |
         echo "$PATH"
         echo "$PYTHONHOME"
-        python --version
-        python -m pip --version
-        python -m pip nose --version || python -m pip install nose
-    - name: Install nose for other Python ${{ matrix.python-version }}
+        $PIP -qq show nose || $PIP install nose
+    - name: Install nose for other Python 2
+      if: ${{ matrix.python-impl == 'jython' || matrix.python-version == '2.6' }}
+      shell: bash
+      run: |
+        # Work around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb)
+        $PIP -qq show nose || { \
+          curl -L -O "https://files.pythonhosted.org/packages/99/4f/13fb671119e65c4dce97c60e67d3fd9e6f7f809f2b307e2611f4701205cb/nose-1.3.7-py2-none-any.whl"; \
+          $PIP install nose-1.3.7-py2-none-any.whl; }
+    - name: Install nose for other Python 3
       if: ${{ matrix.python-version == '3.2' && steps.setup-python.outputs.python-path }}
       shell: bash
       run: |
-        python -m pip nose --version || { \
+        $PIP -qq show nose || { \
           curl -L -O "https://files.pythonhosted.org/packages/15/d8/dd071918c040f50fa1cf80da16423af51ff8ce4a0f2399b7bf8de45ac3d9/nose-1.3.7-py3-none-any.whl"; \
-          python --version; \
-          printf '%s\n' \
-            'import sys' \
-            'print(sys.path)' \
-            | python -; \
-          python -m pip --version; \
-          python -m pip install nose-1.3.7-py3-none-any.whl; }
-    - name: Install nose (Jython)
-      if: ${{ matrix.python-impl == 'jython' }}
-      # Work around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb)
-      run: |
-        pip nose --version || { \
-          curl -L -O "https://files.pythonhosted.org/packages/99/4f/13fb671119e65c4dce97c60e67d3fd9e6f7f809f2b307e2611f4701205cb/nose-1.3.7-py2-none-any.whl"; \
-          pip --version; \
-          pip install nose-1.3.7-py2-none-any.whl; }
+          $PIP install nose-1.3.7-py3-none-any.whl; }
     - name: Set up nosetest test
       if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }}
       shell: bash
@@ -321,7 +345,11 @@ jobs:
         # define a test to validate the Python version used by nosetests
         printf '%s\n' \
           'from __future__ import unicode_literals' \
-          'import sys, os, platform, unittest' \
+          'import sys, os, platform' \
+          'try:' \
+          '    import unittest2 as unittest' \
+          'except ImportError:' \
+          '    import unittest' \
           'class TestPython(unittest.TestCase):' \
           '    def setUp(self):' \
           '        self.ver = os.environ["PYTHON_VER"].split("-")' \
@@ -340,7 +368,6 @@ jobs:
         YTDL_TEST_SET: ${{ matrix.ytdl-test-set }}
         PYTHON_VER: ${{ matrix.python-version }}
         PYTHON_IMPL: ${{ matrix.python-impl }}
-
       run: |
         ./devscripts/run_tests.${{ matrix.run-tests-ext }}
 
diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py
index 4bddca047..a8b6ff1b9 100644
--- a/devscripts/make_lazy_extractors.py
+++ b/devscripts/make_lazy_extractors.py
@@ -6,10 +6,6 @@ import os
 from os.path import dirname as dirn
 import sys
 
-from youtube_dl.compat import compat_register_utf8
-
-compat_register_utf8()
-
 print('WARNING: Lazy loading extractors is an experimental feature that may not always work', file=sys.stderr)
 
 sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
@@ -23,6 +19,10 @@ try:
 except OSError:
     pass
 
+from youtube_dl.compat import compat_register_utf8
+
+compat_register_utf8()
+
 from youtube_dl.extractor import _ALL_CLASSES
 from youtube_dl.extractor.common import InfoExtractor, SearchInfoExtractor
 
diff --git a/test/test_execution.py b/test/test_execution.py
index 1dee53a0f..35e7a5651 100644
--- a/test/test_execution.py
+++ b/test/test_execution.py
@@ -11,13 +11,12 @@ import subprocess
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from youtube_dl.compat import compat_register_utf8
-
 from youtube_dl.utils import encodeArgument
 
-rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
-
 compat_register_utf8()
 
+rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
 try:
     _DEV_NULL = subprocess.DEVNULL
 except AttributeError:
@@ -33,21 +32,22 @@ class TestExecution(unittest.TestCase):
         subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)
 
     def test_main_exec(self):
-        subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+        subprocess.check_call([sys.executable, os.path.normpath('youtube_dl/__main__.py'), '--version'], cwd=rootDir, stdout=_DEV_NULL)
 
     @unittest.skipIf(sys.version_info < (2, 7), 'Python 2.6 doesn\'t support package execution')
     def test_cmdline_umlauts(self):
+        os.environ['PYTHONIOENCODING'] = 'utf-8'
         p = subprocess.Popen(
-            [sys.executable, 'youtube_dl/__main__.py', encodeArgument('ä'), '--version'],
+            [sys.executable, os.path.normpath('youtube_dl/__main__.py'), encodeArgument('ä'), '--version'],
             cwd=rootDir, stdout=_DEV_NULL, stderr=subprocess.PIPE)
         _, stderr = p.communicate()
         self.assertFalse(stderr)
 
     def test_lazy_extractors(self):
-        lazy_extractors = 'youtube_dl/extractor/lazy_extractors.py'
+        lazy_extractors = os.path.normpath('youtube_dl/extractor/lazy_extractors.py')
         try:
-            subprocess.check_call([sys.executable, 'devscripts/make_lazy_extractors.py', lazy_extractors], cwd=rootDir, stdout=_DEV_NULL)
-            subprocess.check_call([sys.executable, 'test/test_all_urls.py'], cwd=rootDir, stdout=_DEV_NULL)
+            subprocess.check_call([sys.executable, os.path.normpath('devscripts/make_lazy_extractors.py'), lazy_extractors], cwd=rootDir, stdout=_DEV_NULL)
+            subprocess.check_call([sys.executable, os.path.normpath('test/test_all_urls.py')], cwd=rootDir, stdout=_DEV_NULL)
         finally:
             for x in ['', 'c'] if sys.version_info[0] < 3 else ['']:
                 try:

From f24bc9272e9b74efc4c4af87c862f5f78921d424 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 4 Jul 2023 16:06:21 +0100
Subject: [PATCH 659/860] [Misc] Fixes for 2.6 compatibility

---
 test/test_jsinterp.py   | 10 ++++++----
 test/test_utils.py      |  2 +-
 youtube_dl/YoutubeDL.py |  6 +++++-
 youtube_dl/compat.py    | 12 ++++++++++++
 youtube_dl/jsinterp.py  | 13 ++++++++++++-
 youtube_dl/utils.py     |  3 ++-
 6 files changed, 38 insertions(+), 8 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index ecd6ab3c9..91b12f544 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -492,10 +492,12 @@ class TestJSInterpreter(unittest.TestCase):
         jsi = JSInterpreter('''
         function x() { let a=/,,[/,913,/](,)}/; "".replace(a, ""); return a; }
         ''')
-        attrs = set(('findall', 'finditer', 'flags', 'groupindex',
-                     'groups', 'match', 'pattern', 'scanner',
-                     'search', 'split', 'sub', 'subn'))
-        self.assertTrue(set(dir(jsi.call_function('x'))) > attrs)
+        attrs = set(('findall', 'finditer', 'match', 'scanner', 'search',
+                     'split', 'sub', 'subn'))
+        if sys.version_info >= (2, 7):
+            # documented for 2.6 but may not be found
+            attrs.update(('flags', 'groupindex', 'groups', 'pattern'))
+        self.assertSetEqual(set(dir(jsi.call_function('x'))) & attrs, attrs)
 
         jsi = JSInterpreter('''
         function x() { let a=/,,[/,913,/](,)}/i; return a; }
diff --git a/test/test_utils.py b/test/test_utils.py
index b85d397d0..5fab05f7c 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1612,7 +1612,7 @@ Line 1
         self.assertEqual(traverse_obj(_TEST_DATA, lambda x, y: x == 'urls' and isinstance(y, list)),
                          [_TEST_DATA['urls']],
                          msg='function as query key should perform a filter based on (key, value)')
-        self.assertCountEqual(traverse_obj(_TEST_DATA, lambda _, x: isinstance(x[0], compat_str)), {'str'},
+        self.assertCountEqual(traverse_obj(_TEST_DATA, lambda _, x: isinstance(x[0], compat_str)), ('str',),
                               msg='exceptions in the query function should be caught')
 
         # Test alternative paths
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 98b878fc1..068029d3e 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -25,7 +25,11 @@ import tokenize
 import traceback
 import random
 
-from ssl import OPENSSL_VERSION
+try:
+    from ssl import OPENSSL_VERSION
+except ImportError:
+    # Must be Python 2.6, should be built against 1.0.2
+    OPENSSL_VERSION = 'OpenSSL 1.0.2(?)'
 from string import ascii_letters
 
 from .compat import (
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 0f4d3756f..2554fd1c3 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -1,10 +1,12 @@
 # coding: utf-8
 from __future__ import unicode_literals
+from __future__ import division
 
 import base64
 import binascii
 import collections
 import ctypes
+import datetime
 import email
 import getpass
 import io
@@ -3150,6 +3152,15 @@ def compat_register_utf8():
             lambda name: lookup('utf-8') if name == 'cp65001' else None)
 
 
+# compat_datetime_timedelta_total_seconds
+try:
+    compat_datetime_timedelta_total_seconds = datetime.timedelta.total_seconds
+except AttributeError:
+    # Py 2.6
+    def compat_datetime_timedelta_total_seconds(td):
+        return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6
+
+
 legacy = [
     'compat_HTMLParseError',
     'compat_HTMLParser',
@@ -3187,6 +3198,7 @@ __all__ = [
     'compat_chr',
     'compat_collections_abc',
     'compat_collections_chain_map',
+    'compat_datetime_timedelta_total_seconds',
     'compat_http_cookiejar',
     'compat_http_cookiejar_Cookie',
     'compat_http_cookies',
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 1ba9c3d67..882432b80 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -277,9 +277,20 @@ class JSInterpreter(object):
 
         def __getattr__(self, name):
             self.__instantiate()
+            # make Py 2.6 conform to its lying documentation
+            if name == 'flags':
+                self.flags = self.__flags
+            elif name == 'pattern':
+                self.pattern = self.__pattern_txt
+            elif name in ('groupindex', 'groups'):
+                # in case these get set after a match?
+                if hasattr(self.__self, name):
+                    setattr(self, name, getattr(self.__self, name))
+                else:
+                    return 0 if name == 'groupindex' else {}
             if hasattr(self, name):
                 return getattr(self, name)
-            return super(JSInterpreter.JS_RegExp, self).__getattr__(name)
+            raise AttributeError('{0} has no attribute named {1}'.format(self, name))
 
         @classmethod
         def regex_flags(cls, expr):
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 584581b6a..83f67bd95 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -47,6 +47,7 @@ from .compat import (
     compat_collections_abc,
     compat_cookiejar,
     compat_ctypes_WINFUNCTYPE,
+    compat_datetime_timedelta_total_seconds,
     compat_etree_fromstring,
     compat_expanduser,
     compat_html_entities,
@@ -3102,7 +3103,7 @@ def unified_timestamp(date_str, day_first=True):
             pass
     timetuple = email.utils.parsedate_tz(date_str)
     if timetuple:
-        return calendar.timegm(timetuple) + pm_delta * 3600 - timezone.total_seconds()
+        return calendar.timegm(timetuple) + pm_delta * 3600 - compat_datetime_timedelta_total_seconds(timezone)
 
 
 def determine_ext(url, default_ext='unknown_video'):

From b6dff4073d469cceadb099c00ccbf3bd6fc515a6 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 7 Jul 2023 18:41:32 +0100
Subject: [PATCH 660/860] [core] Revert version display from b8a86dc

---
 youtube_dl/YoutubeDL.py | 8 +++++---
 1 file changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 068029d3e..4e7fd1063 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -2378,10 +2378,12 @@ class YoutubeDL(object):
                 self.get_encoding()))
         write_string(encoding_str, encoding=None)
 
-        self._write_string('[debug] youtube-dl version ' + __version__ + (' (single file build)\n' if ytdl_is_updateable() else '\n'))
+        writeln_debug = lambda *s: self._write_string('[debug] %s\n' % (''.join(s), ))
+        writeln_debug('youtube-dl version ', __version__)
         if _LAZY_LOADER:
-            self._write_string('[debug] Lazy loading extractors enabled\n')
-        writeln_debug = lambda *s: self._write_string('[debug] %s\n' % (''.join(s), ))  # moved down for easier merge
+            writeln_debug('Lazy loading extractors enabled')
+        if ytdl_is_updateable():
+            writeln_debug('Single file build')
         try:
             sp = subprocess.Popen(
                 ['git', 'rev-parse', '--short', 'HEAD'],

From f47fdb9564d3ca1c0fa70ed6031148ec908fdc7b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 6 Jul 2023 15:46:22 +0100
Subject: [PATCH 661/860] [utils] Add {expected_type} and Iterable support to
 traverse_obj()

---
 test/test_utils.py  | 153 ++++++++++++++++++++++++++------
 youtube_dl/utils.py | 211 +++++++++++++++++++++++++++++---------------
 2 files changed, 265 insertions(+), 99 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 5fab05f7c..1fc16ed05 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -79,10 +79,12 @@ from youtube_dl.utils import (
     rot47,
     shell_quote,
     smuggle_url,
+    str_or_none,
     str_to_int,
     strip_jsonp,
     strip_or_none,
     subtitles_filename,
+    T,
     timeconvert,
     traverse_obj,
     try_call,
@@ -1566,6 +1568,7 @@ Line 1
         self.assertEqual(variadic('spam', allowed_types=[dict]), 'spam')
 
     def test_traverse_obj(self):
+        str = compat_str
         _TEST_DATA = {
             100: 100,
             1.2: 1.2,
@@ -1598,8 +1601,8 @@ Line 1
 
         # Test Ellipsis behavior
         self.assertCountEqual(traverse_obj(_TEST_DATA, Ellipsis),
-                              (item for item in _TEST_DATA.values() if item is not None),
-                              msg='`...` should give all values except `None`')
+                              (item for item in _TEST_DATA.values() if item not in (None, {})),
+                              msg='`...` should give all non discarded values')
         self.assertCountEqual(traverse_obj(_TEST_DATA, ('urls', 0, Ellipsis)), _TEST_DATA['urls'][0].values(),
                               msg='`...` selection for dicts should select all values')
         self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, Ellipsis, 'url')),
@@ -1607,13 +1610,51 @@ Line 1
                          msg='nested `...` queries should work')
         self.assertCountEqual(traverse_obj(_TEST_DATA, (Ellipsis, Ellipsis, 'index')), range(4),
                               msg='`...` query result should be flattened')
+        self.assertEqual(traverse_obj(iter(range(4)), Ellipsis), list(range(4)),
+                         msg='`...` should accept iterables')
 
         # Test function as key
         self.assertEqual(traverse_obj(_TEST_DATA, lambda x, y: x == 'urls' and isinstance(y, list)),
                          [_TEST_DATA['urls']],
                          msg='function as query key should perform a filter based on (key, value)')
-        self.assertCountEqual(traverse_obj(_TEST_DATA, lambda _, x: isinstance(x[0], compat_str)), ('str',),
-                              msg='exceptions in the query function should be caught')
+        self.assertCountEqual(traverse_obj(_TEST_DATA, lambda _, x: isinstance(x[0], str)), {'str'},
+                              msg='exceptions in the query function should be catched')
+        self.assertEqual(traverse_obj(iter(range(4)), lambda _, x: x % 2 == 0), [0, 2],
+                         msg='function key should accept iterables')
+        if __debug__:
+            with self.assertRaises(Exception, msg='Wrong function signature should raise in debug'):
+                traverse_obj(_TEST_DATA, lambda a: Ellipsis)
+            with self.assertRaises(Exception, msg='Wrong function signature should raise in debug'):
+                traverse_obj(_TEST_DATA, lambda a, b, c: Ellipsis)
+
+        # Test set as key (transformation/type, like `expected_type`)
+        self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str.upper), )), ['STR'],
+                         msg='Function in set should be a transformation')
+        self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str))), ['str'],
+                         msg='Type in set should be a type filter')
+        self.assertEqual(traverse_obj(_TEST_DATA, T(dict)), _TEST_DATA,
+                         msg='A single set should be wrapped into a path')
+        self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str.upper))), ['STR'],
+                         msg='Transformation function should not raise')
+        self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str_or_none))),
+                         [item for item in map(str_or_none, _TEST_DATA.values()) if item is not None],
+                         msg='Function in set should be a transformation')
+        if __debug__:
+            with self.assertRaises(Exception, msg='Sets with length != 1 should raise in debug'):
+                traverse_obj(_TEST_DATA, set())
+            with self.assertRaises(Exception, msg='Sets with length != 1 should raise in debug'):
+                traverse_obj(_TEST_DATA, {str.upper, str})
+
+        # Test `slice` as a key
+        _SLICE_DATA = [0, 1, 2, 3, 4]
+        self.assertEqual(traverse_obj(_TEST_DATA, ('dict', slice(1))), None,
+                         msg='slice on a dictionary should not throw')
+        self.assertEqual(traverse_obj(_SLICE_DATA, slice(1)), _SLICE_DATA[:1],
+                         msg='slice key should apply slice to sequence')
+        self.assertEqual(traverse_obj(_SLICE_DATA, slice(1, 2)), _SLICE_DATA[1:2],
+                         msg='slice key should apply slice to sequence')
+        self.assertEqual(traverse_obj(_SLICE_DATA, slice(1, 4, 2)), _SLICE_DATA[1:4:2],
+                         msg='slice key should apply slice to sequence')
 
         # Test alternative paths
         self.assertEqual(traverse_obj(_TEST_DATA, 'fail', 'str'), 'str',
@@ -1659,15 +1700,23 @@ Line 1
                          {0: ['https://www.example.com/1', 'https://www.example.com/0']},
                          msg='triple nesting in dict path should be treated as branches')
         self.assertEqual(traverse_obj(_TEST_DATA, {0: 'fail'}), {},
-                         msg='remove `None` values when dict key')
+                         msg='remove `None` values when top level dict key fails')
         self.assertEqual(traverse_obj(_TEST_DATA, {0: 'fail'}, default=Ellipsis), {0: Ellipsis},
-                         msg='do not remove `None` values if `default`')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}), {0: {}},
-                         msg='do not remove empty values when dict key')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}, default=Ellipsis), {0: {}},
-                         msg='do not remove empty values when dict key and a default')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('dict', Ellipsis)}), {0: []},
-                         msg='if branch in dict key not successful, return `[]`')
+                         msg='use `default` if key fails and `default`')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}), {},
+                         msg='remove empty values when dict key')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}, default=Ellipsis), {0: Ellipsis},
+                         msg='use `default` when dict key and `default`')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 'fail'}}), {},
+                         msg='remove empty values when nested dict key fails')
+        self.assertEqual(traverse_obj(None, {0: 'fail'}), {},
+                         msg='default to dict if pruned')
+        self.assertEqual(traverse_obj(None, {0: 'fail'}, default=Ellipsis), {0: Ellipsis},
+                         msg='default to dict if pruned and default is given')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 'fail'}}, default=Ellipsis), {0: {0: Ellipsis}},
+                         msg='use nested `default` when nested dict key fails and `default`')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('dict', Ellipsis)}), {},
+                         msg='remove key if branch in dict key not successful')
 
         # Testing default parameter behavior
         _DEFAULT_DATA = {'None': None, 'int': 0, 'list': []}
@@ -1691,20 +1740,55 @@ Line 1
                          msg='if branched but not successful return `[]`, not `default`')
         self.assertEqual(traverse_obj(_DEFAULT_DATA, ('list', Ellipsis)), [],
                          msg='if branched but object is empty return `[]`, not `default`')
+        self.assertEqual(traverse_obj(None, Ellipsis), [],
+                         msg='if branched but object is `None` return `[]`, not `default`')
+        self.assertEqual(traverse_obj({0: None}, (0, Ellipsis)), [],
+                         msg='if branched but state is `None` return `[]`, not `default`')
+
+        branching_paths = [
+            ('fail', Ellipsis),
+            (Ellipsis, 'fail'),
+            100 * ('fail',) + (Ellipsis,),
+            (Ellipsis,) + 100 * ('fail',),
+        ]
+        for branching_path in branching_paths:
+            self.assertEqual(traverse_obj({}, branching_path), [],
+                             msg='if branched but state is `None`, return `[]` (not `default`)')
+            self.assertEqual(traverse_obj({}, 'fail', branching_path), [],
+                             msg='if branching in last alternative and previous did not match, return `[]` (not `default`)')
+            self.assertEqual(traverse_obj({0: 'x'}, 0, branching_path), 'x',
+                             msg='if branching in last alternative and previous did match, return single value')
+            self.assertEqual(traverse_obj({0: 'x'}, branching_path, 0), 'x',
+                             msg='if branching in first alternative and non-branching path does match, return single value')
+            self.assertEqual(traverse_obj({}, branching_path, 'fail'), None,
+                             msg='if branching in first alternative and non-branching path does not match, return `default`')
 
         # Testing expected_type behavior
         _EXPECTED_TYPE_DATA = {'str': 'str', 'int': 0}
-        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=compat_str), 'str',
-                         msg='accept matching `expected_type` type')
-        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=int), None,
-                         msg='reject non matching `expected_type` type')
-        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'int', expected_type=lambda x: compat_str(x)), '0',
-                         msg='transform type using type function')
-        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str',
-                                      expected_type=lambda _: 1 / 0), None,
-                         msg='wrap expected_type function in try_call')
-        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, Ellipsis, expected_type=compat_str), ['str'],
-                         msg='eliminate items that expected_type fails on')
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=str),
+                         'str', msg='accept matching `expected_type` type')
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=int),
+                         None, msg='reject non matching `expected_type` type')
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'int', expected_type=lambda x: str(x)),
+                         '0', msg='transform type using type function')
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=lambda _: 1 / 0),
+                         None, msg='wrap expected_type function in try_call')
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, Ellipsis, expected_type=str),
+                         ['str'], msg='eliminate items that expected_type fails on')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2}, expected_type=int),
+                         {0: 100}, msg='type as expected_type should filter dict values')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2, 2: 'None'}, expected_type=str_or_none),
+                         {0: '100', 1: '1.2'}, msg='function as expected_type should transform dict values')
+        self.assertEqual(traverse_obj(_TEST_DATA, ({0: 1.2}, 0, {int_or_none}), expected_type=int),
+                         1, msg='expected_type should not filter non final dict values')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 100, 1: 'str'}}, expected_type=int),
+                         {0: {0: 100}}, msg='expected_type should transform deep dict values')
+        self.assertEqual(traverse_obj(_TEST_DATA, [({0: '...'}, {0: '...'})], expected_type=type(Ellipsis)),
+                         [{0: Ellipsis}, {0: Ellipsis}], msg='expected_type should transform branched dict values')
+        self.assertEqual(traverse_obj({1: {3: 4}}, [(1, 2), 3], expected_type=int),
+                         [4], msg='expected_type regression for type matching in tuple branching')
+        self.assertEqual(traverse_obj(_TEST_DATA, ['data', Ellipsis], expected_type=int),
+                         [], msg='expected_type regression for type matching in dict result')
 
         # Test get_all behavior
         _GET_ALL_DATA = {'key': [0, 1, 2]}
@@ -1749,14 +1833,23 @@ Line 1
                                       _traverse_string=True), '.',
                          msg='traverse into converted data if `traverse_string`')
         self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', Ellipsis),
-                                      _traverse_string=True), list('str'),
-                         msg='`...` branching into string should result in list')
+                                      _traverse_string=True), 'str',
+                         msg='`...` should result in string (same value) if `traverse_string`')
+        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', slice(0, None, 2)),
+                                      _traverse_string=True), 'sr',
+                         msg='`slice` should result in string if `traverse_string`')
+        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', lambda i, v: i or v == "s"),
+                                      _traverse_string=True), 'str',
+                         msg='function should result in string if `traverse_string`')
         self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', (0, 2)),
                                       _traverse_string=True), ['s', 'r'],
-                         msg='branching into string should result in list')
-        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', lambda _, x: x),
-                                      _traverse_string=True), list('str'),
-                         msg='function branching into string should result in list')
+                         msg='branching should result in list if `traverse_string`')
+        self.assertEqual(traverse_obj({}, (0, Ellipsis), _traverse_string=True), [],
+                         msg='branching should result in list if `traverse_string`')
+        self.assertEqual(traverse_obj({}, (0, lambda x, y: True), _traverse_string=True), [],
+                         msg='branching should result in list if `traverse_string`')
+        self.assertEqual(traverse_obj({}, (0, slice(1)), _traverse_string=True), [],
+                         msg='branching should result in list if `traverse_string`')
 
         # Test is_user_input behavior
         _IS_USER_INPUT_DATA = {'range8': list(range(8))}
@@ -1793,6 +1886,8 @@ Line 1
                          msg='failing str key on a `re.Match` should return `default`')
         self.assertEqual(traverse_obj(mobj, 8), None,
                          msg='failing int key on a `re.Match` should return `default`')
+        self.assertEqual(traverse_obj(mobj, lambda k, _: k in (0, 'group')), ['0123', '3'],
+                         msg='function on a `re.Match` should give group name as well')
 
     def test_get_first(self):
         self.assertEqual(get_first([{'a': None}, {'a': 'spam'}], 'a'), 'spam')
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 83f67bd95..dbdbe5f59 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -16,6 +16,7 @@ import email.header
 import errno
 import functools
 import gzip
+import inspect
 import io
 import itertools
 import json
@@ -3881,7 +3882,7 @@ def detect_exe_version(output, version_re=None, unrecognized='present'):
         return unrecognized
 
 
-class LazyList(compat_collections_abc.Sequence):
+class LazyList(compat_collections_abc.Iterable):
     """Lazy immutable list from an iterable
     Note that slices of a LazyList are lists and not LazyList"""
 
@@ -4223,10 +4224,16 @@ def multipart_encode(data, boundary=None):
     return out, content_type
 
 
-def variadic(x, allowed_types=(compat_str, bytes, dict)):
-    if not isinstance(allowed_types, tuple) and isinstance(allowed_types, compat_collections_abc.Iterable):
+def is_iterable_like(x, allowed_types=compat_collections_abc.Iterable, blocked_types=NO_DEFAULT):
+    if blocked_types is NO_DEFAULT:
+        blocked_types = (compat_str, bytes, compat_collections_abc.Mapping)
+    return isinstance(x, allowed_types) and not isinstance(x, blocked_types)
+
+
+def variadic(x, allowed_types=NO_DEFAULT):
+    if isinstance(allowed_types, compat_collections_abc.Iterable):
         allowed_types = tuple(allowed_types)
-    return x if isinstance(x, compat_collections_abc.Iterable) and not isinstance(x, allowed_types) else (x,)
+    return x if is_iterable_like(x, blocked_types=allowed_types) else (x,)
 
 
 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
@@ -5993,7 +6000,7 @@ def clean_podcast_url(url):
 
 def traverse_obj(obj, *paths, **kwargs):
     """
-    Safely traverse nested `dict`s and `Sequence`s
+    Safely traverse nested `dict`s and `Iterable`s
 
     >>> obj = [{}, {"key": "value"}]
     >>> traverse_obj(obj, (1, "key"))
@@ -6001,14 +6008,17 @@ def traverse_obj(obj, *paths, **kwargs):
 
     Each of the provided `paths` is tested and the first producing a valid result will be returned.
     The next path will also be tested if the path branched but no results could be found.
-    Supported values for traversal are `Mapping`, `Sequence` and `re.Match`.
-    A value of None is treated as the absence of a value.
+    Supported values for traversal are `Mapping`, `Iterable` and `re.Match`.
+    Unhelpful values (`{}`, `None`) are treated as the absence of a value and discarded.
 
     The paths will be wrapped in `variadic`, so that `'key'` is conveniently the same as `('key', )`.
 
     The keys in the path can be one of:
         - `None`:           Return the current object.
-        - `str`/`int`:      Return `obj[key]`. For `re.Match, return `obj.group(key)`.
+        - `set`:            Requires the only item in the set to be a type or function,
+                            like `{type}`/`{func}`. If a `type`, returns only values
+                            of this type. If a function, returns `func(obj)`.
+        - `str`/`int`:      Return `obj[key]`. For `re.Match`, return `obj.group(key)`.
         - `slice`:          Branch out and return all values in `obj[key]`.
         - `Ellipsis`:       Branch out and return a list of all values.
         - `tuple`/`list`:   Branch out and return a list of all matching values.
@@ -6016,6 +6026,9 @@ def traverse_obj(obj, *paths, **kwargs):
         - `function`:       Branch out and return values filtered by the function.
                             Read as: `[value for key, value in obj if function(key, value)]`.
                             For `Sequence`s, `key` is the index of the value.
+                            For `Iterable`s, `key` is the enumeration count of the value.
+                            For `re.Match`es, `key` is the group number (0 = full match)
+                            as well as additionally any group names, if given.
         - `dict`            Transform the current object and return a matching dict.
                             Read as: `{key: traverse_obj(obj, path) for key, path in dct.items()}`.
 
@@ -6024,8 +6037,12 @@ def traverse_obj(obj, *paths, **kwargs):
     @params paths           Paths which to traverse by.
     Keyword arguments:
     @param default          Value to return if the paths do not match.
+                            If the last key in the path is a `dict`, it will apply to each value inside
+                            the dict instead, depth first. Try to avoid if using nested `dict` keys.
     @param expected_type    If a `type`, only accept final values of this type.
                             If any other callable, try to call the function on each result.
+                            If the last key in the path is a `dict`, it will apply to each value inside
+                            the dict instead, recursively. This does respect branching paths.
     @param get_all          If `False`, return the first matching result, otherwise all matching ones.
     @param casesense        If `False`, consider string dictionary keys as case insensitive.
 
@@ -6036,12 +6053,15 @@ def traverse_obj(obj, *paths, **kwargs):
     @param _traverse_string  Whether to traverse into objects as strings.
                             If `True`, any non-compatible object will first be
                             converted into a string and then traversed into.
+                            The return value of that path will be a string instead,
+                            not respecting any further branching.
 
 
     @returns                The result of the object traversal.
                             If successful, `get_all=True`, and the path branches at least once,
                             then a list of results is returned instead.
                             A list is always returned if the last path branches and no `default` is given.
+                            If a path ends on a `dict` that result will always be a `dict`.
     """
 
     # parameter defaults
@@ -6055,7 +6075,6 @@ def traverse_obj(obj, *paths, **kwargs):
     # instant compat
     str = compat_str
 
-    is_sequence = lambda x: isinstance(x, compat_collections_abc.Sequence) and not isinstance(x, (str, bytes))
     casefold = lambda k: compat_casefold(k) if isinstance(k, str) else k
 
     if isinstance(expected_type, type):
@@ -6063,128 +6082,180 @@ def traverse_obj(obj, *paths, **kwargs):
     else:
         type_test = lambda val: try_call(expected_type or IDENTITY, args=(val,))
 
+    def lookup_or_none(v, k, getter=None):
+        try:
+            return getter(v, k) if getter else v[k]
+        except IndexError:
+            return None
+
     def from_iterable(iterables):
         # chain.from_iterable(['ABC', 'DEF']) --> A B C D E F
         for it in iterables:
             for item in it:
                 yield item
 
-    def apply_key(key, obj):
-        if obj is None:
-            return
+    def apply_key(key, obj, is_last):
+        branching = False
+
+        if obj is None and _traverse_string:
+            if key is Ellipsis or callable(key) or isinstance(key, slice):
+                branching = True
+                result = ()
+            else:
+                result = None
 
         elif key is None:
-            yield obj
+            result = obj
+
+        elif isinstance(key, set):
+            assert len(key) == 1, 'Set should only be used to wrap a single item'
+            item = next(iter(key))
+            if isinstance(item, type):
+                result = obj if isinstance(obj, item) else None
+            else:
+                result = try_call(item, args=(obj,))
 
         elif isinstance(key, (list, tuple)):
-            for branch in key:
-                _, result = apply_path(obj, branch)
-                for item in result:
-                    yield item
+            branching = True
+            result = from_iterable(
+                apply_path(obj, branch, is_last)[0] for branch in key)
 
         elif key is Ellipsis:
-            result = []
+            branching = True
             if isinstance(obj, compat_collections_abc.Mapping):
                 result = obj.values()
-            elif is_sequence(obj):
+            elif is_iterable_like(obj):
                 result = obj
             elif isinstance(obj, compat_re_Match):
                 result = obj.groups()
             elif _traverse_string:
+                branching = False
                 result = str(obj)
-            for item in result:
-                yield item
+            else:
+                result = ()
 
         elif callable(key):
-            if is_sequence(obj):
-                iter_obj = enumerate(obj)
-            elif isinstance(obj, compat_collections_abc.Mapping):
+            branching = True
+            if isinstance(obj, compat_collections_abc.Mapping):
                 iter_obj = obj.items()
+            elif is_iterable_like(obj):
+                iter_obj = enumerate(obj)
             elif isinstance(obj, compat_re_Match):
-                iter_obj = enumerate(itertools.chain([obj.group()], obj.groups()))
+                iter_obj = itertools.chain(
+                    enumerate(itertools.chain((obj.group(),), obj.groups())),
+                    obj.groupdict().items())
             elif _traverse_string:
+                branching = False
                 iter_obj = enumerate(str(obj))
             else:
-                return
-            for item in (v for k, v in iter_obj if try_call(key, args=(k, v))):
-                yield item
+                iter_obj = ()
+
+            result = (v for k, v in iter_obj if try_call(key, args=(k, v)))
+            if not branching:  # string traversal
+                result = ''.join(result)
 
         elif isinstance(key, dict):
-            iter_obj = ((k, _traverse_obj(obj, v)) for k, v in key.items())
-            yield dict((k, v if v is not None else default) for k, v in iter_obj
-                       if v is not None or default is not NO_DEFAULT)
+            iter_obj = ((k, _traverse_obj(obj, v, False, is_last)) for k, v in key.items())
+            result = dict((k, v if v is not None else default) for k, v in iter_obj
+                          if v is not None or default is not NO_DEFAULT) or None
 
         elif isinstance(obj, compat_collections_abc.Mapping):
-            yield (obj.get(key) if casesense or (key in obj)
-                   else next((v for k, v in obj.items() if casefold(k) == key), None))
+            result = (try_call(obj.get, args=(key,))
+                      if casesense or try_call(obj.__contains__, args=(key,))
+                      else next((v for k, v in obj.items() if casefold(k) == key), None))
 
         elif isinstance(obj, compat_re_Match):
+            result = None
             if isinstance(key, int) or casesense:
-                try:
-                    yield obj.group(key)
-                    return
-                except IndexError:
-                    pass
-            if not isinstance(key, str):
-                return
+                result = lookup_or_none(obj, key, getter=compat_re_Match.group)
 
-            yield next((v for k, v in obj.groupdict().items() if casefold(k) == key), None)
+            elif isinstance(key, str):
+                result = next((v for k, v in obj.groupdict().items()
+                              if casefold(k) == key), None)
 
         else:
-            if _is_user_input:
-                key = (int_or_none(key) if ':' not in key
-                       else slice(*map(int_or_none, key.split(':'))))
+            result = None
+            if isinstance(key, (int, slice)):
+                if is_iterable_like(obj, compat_collections_abc.Sequence):
+                    branching = isinstance(key, slice)
+                    result = lookup_or_none(obj, key)
+                elif _traverse_string:
+                    result = lookup_or_none(str(obj), key)
 
-            if not isinstance(key, (int, slice)):
-                return
+        return branching, result if branching else (result,)
 
-            if not is_sequence(obj):
-                if not _traverse_string:
-                    return
-                obj = str(obj)
+    def lazy_last(iterable):
+        iterator = iter(iterable)
+        prev = next(iterator, NO_DEFAULT)
+        if prev is NO_DEFAULT:
+            return
 
-            try:
-                yield obj[key]
-            except IndexError:
-                pass
+        for item in iterator:
+            yield False, prev
+            prev = item
 
-    def apply_path(start_obj, path):
+        yield True, prev
+
+    def apply_path(start_obj, path, test_type):
         objs = (start_obj,)
         has_branched = False
 
-        for key in variadic(path):
-            if _is_user_input and key == ':':
-                key = Ellipsis
+        key = None
+        for last, key in lazy_last(variadic(path, (str, bytes, dict, set))):
+            if _is_user_input and isinstance(key, str):
+                if key == ':':
+                    key = Ellipsis
+                elif ':' in key:
+                    key = slice(*map(int_or_none, key.split(':')))
+                elif int_or_none(key) is not None:
+                    key = int(key)
 
             if not casesense and isinstance(key, str):
                 key = compat_casefold(key)
 
-            if key is Ellipsis or isinstance(key, (list, tuple)) or callable(key):
-                has_branched = True
+            if __debug__ and callable(key):
+                # Verify function signature
+                inspect.getcallargs(key, None, None)
 
-            key_func = functools.partial(apply_key, key)
-            objs = from_iterable(map(key_func, objs))
+            new_objs = []
+            for obj in objs:
+                branching, results = apply_key(key, obj, last)
+                has_branched |= branching
+                new_objs.append(results)
 
-        return has_branched, objs
+            objs = from_iterable(new_objs)
 
-    def _traverse_obj(obj, path, use_list=True):
-        has_branched, results = apply_path(obj, path)
-        results = LazyList(x for x in map(type_test, results) if x is not None)
+        if test_type and not isinstance(key, (dict, list, tuple)):
+            objs = map(type_test, objs)
+
+        return objs, has_branched, isinstance(key, dict)
+
+    def _traverse_obj(obj, path, allow_empty, test_type):
+        results, has_branched, is_dict = apply_path(obj, path, test_type)
+        results = LazyList(x for x in results if x not in (None, {}))
 
         if get_all and has_branched:
-            return results.exhaust() if results or use_list else None
+            if results:
+                return results.exhaust()
+            if allow_empty:
+                return [] if default is NO_DEFAULT else default
+            return None
 
-        return results[0] if results else None
+        return results[0] if results else {} if allow_empty and is_dict else None
 
     for index, path in enumerate(paths, 1):
-        use_list = default is NO_DEFAULT and index == len(paths)
-        result = _traverse_obj(obj, path, use_list)
+        result = _traverse_obj(obj, path, index == len(paths), True)
         if result is not None:
             return result
 
     return None if default is NO_DEFAULT else default
 
 
+def T(x):
+    """ For use in yt-dl instead of {type} or set((type,)) """
+    return set((x,))
+
+
 def get_first(obj, keys, **kwargs):
     return traverse_obj(obj, (Ellipsis,) + tuple(variadic(keys)), get_all=False, **kwargs)
 

From d5ef405c5d533c85cebd205a5b7958614c7013f3 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 7 Jul 2023 18:45:31 +0100
Subject: [PATCH 662/860] [core] Align error reporting methods with yt-dlp

---
 test/helper.py          |  3 ++-
 test/test_YoutubeDL.py  | 10 ++--------
 youtube_dl/YoutubeDL.py | 39 ++++++++++++++++++++++++++++++++-------
 3 files changed, 36 insertions(+), 16 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 883b2e877..e3314b03e 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -72,7 +72,8 @@ class FakeYDL(YoutubeDL):
     def to_screen(self, s, skip_eol=None):
         print(s)
 
-    def trouble(self, s, tb=None):
+    def trouble(self, *args, **kwargs):
+        s = args[0] if len(args) > 0 else kwargs.get('message', 'Missing message')
         raise Exception(s)
 
     def download(self, x):
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index f8c8e619c..60780b8a7 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -930,17 +930,11 @@ class TestYoutubeDL(unittest.TestCase):
     # Test case for https://github.com/ytdl-org/youtube-dl/issues/27064
     def test_ignoreerrors_for_playlist_with_url_transparent_iterable_entries(self):
 
-        class _YDL(YDL):
-            def __init__(self, *args, **kwargs):
-                super(_YDL, self).__init__(*args, **kwargs)
-
-            def trouble(self, s, tb=None):
-                pass
-
-        ydl = _YDL({
+        ydl = YDL({
             'format': 'extra',
             'ignoreerrors': True,
         })
+        ydl.trouble = lambda *_, **__: None
 
         class VideoIE(InfoExtractor):
             _VALID_URL = r'video:(?P<id>\d+)'
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 4e7fd1063..1435754c2 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -582,7 +582,7 @@ class YoutubeDL(object):
         if self.params.get('cookiefile') is not None:
             self.cookiejar.save(ignore_discard=True, ignore_expires=True)
 
-    def trouble(self, message=None, tb=None):
+    def trouble(self, *args, **kwargs):
         """Determine action to take when a download problem appears.
 
         Depending on if the downloader has been configured to ignore
@@ -591,6 +591,11 @@ class YoutubeDL(object):
 
         tb, if given, is additional traceback information.
         """
+        # message=None, tb=None, is_error=True
+        message = args[0] if len(args) > 0 else kwargs.get('message', None)
+        tb = args[1] if len(args) > 1 else kwargs.get('tb', None)
+        is_error = args[2] if len(args) > 2 else kwargs.get('is_error', True)
+
         if message is not None:
             self.to_stderr(message)
         if self.params.get('verbose'):
@@ -603,7 +608,10 @@ class YoutubeDL(object):
                 else:
                     tb_data = traceback.format_list(traceback.extract_stack())
                     tb = ''.join(tb_data)
-            self.to_stderr(tb)
+            if tb:
+                self.to_stderr(tb)
+        if not is_error:
+            return
         if not self.params.get('ignoreerrors', False):
             if sys.exc_info()[0] and hasattr(sys.exc_info()[1], 'exc_info') and sys.exc_info()[1].exc_info[0]:
                 exc_info = sys.exc_info()[1].exc_info
@@ -612,11 +620,18 @@ class YoutubeDL(object):
             raise DownloadError(message, exc_info)
         self._download_retcode = 1
 
-    def report_warning(self, message):
+    def report_warning(self, message, only_once=False, _cache={}):
         '''
         Print the message to stderr, it will be prefixed with 'WARNING:'
         If stderr is a tty file the 'WARNING:' will be colored
         '''
+        if only_once:
+            m_hash = hash((self, message))
+            m_cnt = _cache.setdefault(m_hash, 0)
+            _cache[m_hash] = m_cnt + 1
+            if m_cnt > 0:
+                return
+
         if self.params.get('logger') is not None:
             self.params['logger'].warning(message)
         else:
@@ -629,7 +644,7 @@ class YoutubeDL(object):
             warning_message = '%s %s' % (_msg_header, message)
             self.to_stderr(warning_message)
 
-    def report_error(self, message, tb=None):
+    def report_error(self, message, *args, **kwargs):
         '''
         Do the same as trouble, but prefixes the message with 'ERROR:', colored
         in red if stderr is a tty file.
@@ -638,8 +653,18 @@ class YoutubeDL(object):
             _msg_header = '\033[0;31mERROR:\033[0m'
         else:
             _msg_header = 'ERROR:'
-        error_message = '%s %s' % (_msg_header, message)
-        self.trouble(error_message, tb)
+        kwargs['message'] = '%s %s' % (_msg_header, message)
+        self.trouble(*args, **kwargs)
+
+    def report_unscoped_cookies(self, *args, **kwargs):
+        # message=None, tb=False, is_error=False
+        if len(args) <= 2:
+            kwargs.setdefault('is_error', False)
+            if len(args) <= 0:
+                kwargs.setdefault(
+                    'message',
+                    'Unscoped cookies are not allowed: please specify some sort of scoping')
+        self.report_error(*args, **kwargs)
 
     def report_file_already_downloaded(self, file_name):
         """Report file has already been fully downloaded."""
@@ -835,7 +860,7 @@ class YoutubeDL(object):
                 msg += '\nYou might want to use a VPN or a proxy server (with --proxy) to workaround.'
                 self.report_error(msg)
             except ExtractorError as e:  # An error we somewhat expected
-                self.report_error(compat_str(e), e.format_traceback())
+                self.report_error(compat_str(e), tb=e.format_traceback())
             except MaxDownloadsReached:
                 raise
             except Exception as e:

From 1720c04dc56fa0d2caa0a455b1acbd569347482e Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 17 Jul 2023 20:47:58 +0100
Subject: [PATCH 663/860] [test] Make skipped tests in test_execution work with
 Py 2.6

---
 test/test_execution.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/test/test_execution.py b/test/test_execution.py
index 35e7a5651..ae59e562a 100644
--- a/test/test_execution.py
+++ b/test/test_execution.py
@@ -24,21 +24,24 @@ except AttributeError:
 
 
 class TestExecution(unittest.TestCase):
+    def setUp(self):
+        self.module = 'youtube_dl'
+        if sys.version_info < (2, 7):
+            self.module += '.__main__'
+
     def test_import(self):
         subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir)
 
-    @unittest.skipIf(sys.version_info < (2, 7), 'Python 2.6 doesn\'t support package execution')
     def test_module_exec(self):
-        subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL)
+        subprocess.check_call([sys.executable, '-m', self.module, '--version'], cwd=rootDir, stdout=_DEV_NULL)
 
     def test_main_exec(self):
         subprocess.check_call([sys.executable, os.path.normpath('youtube_dl/__main__.py'), '--version'], cwd=rootDir, stdout=_DEV_NULL)
 
-    @unittest.skipIf(sys.version_info < (2, 7), 'Python 2.6 doesn\'t support package execution')
     def test_cmdline_umlauts(self):
         os.environ['PYTHONIOENCODING'] = 'utf-8'
         p = subprocess.Popen(
-            [sys.executable, os.path.normpath('youtube_dl/__main__.py'), encodeArgument('ä'), '--version'],
+            [sys.executable, '-m', self.module, encodeArgument('ä'), '--version'],
             cwd=rootDir, stdout=_DEV_NULL, stderr=subprocess.PIPE)
         _, stderr = p.communicate()
         self.assertFalse(stderr)

From 648dc5304cb2476592ff142988b8c62675011fcc Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 7 Jul 2023 18:51:38 +0100
Subject: [PATCH 664/860] [compat] Add Request and HTTPClient compat for
 redirect

* support `method` parameter of `Request.__init__`  (Py 2 and old Py 3)
* support `getcode` method of compat_http_client.HTTPResponse (Py 2)
---
 youtube_dl/compat.py | 28 ++++++++++++++++++++++++++++
 1 file changed, 28 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 2554fd1c3..cd11ba5aa 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -21,6 +21,7 @@ import socket
 import struct
 import subprocess
 import sys
+import types
 import xml.etree.ElementTree
 
 # naming convention
@@ -55,6 +56,22 @@ try:
 except ImportError:  # Python 2
     import urllib2 as compat_urllib_request
 
+# Also fix up lack of method arg in old Pythons
+try:
+    _req = compat_urllib_request.Request
+    _req('http://127.0.0.1', method='GET')
+except TypeError:
+    class _request(object):
+        def __new__(cls, url, *args, **kwargs):
+            method = kwargs.pop('method', None)
+            r = _req(url, *args, **kwargs)
+            if method:
+                r.get_method = types.MethodType(lambda _: method, r)
+            return r
+
+    compat_urllib_request.Request = _request
+
+
 try:
     import urllib.error as compat_urllib_error
 except ImportError:  # Python 2
@@ -79,6 +96,12 @@ try:
 except ImportError:  # Python 2
     import urllib as compat_urllib_response
 
+try:
+    compat_urllib_response.addinfourl.status
+except AttributeError:
+    # .getcode() is deprecated in Py 3.
+    compat_urllib_response.addinfourl.status = property(lambda self: self.getcode())
+
 try:
     import http.cookiejar as compat_cookiejar
 except ImportError:  # Python 2
@@ -2360,6 +2383,11 @@ try:
     import http.client as compat_http_client
 except ImportError:  # Python 2
     import httplib as compat_http_client
+try:
+    compat_http_client.HTTPResponse.getcode
+except AttributeError:
+    # Py < 3.1
+    compat_http_client.HTTPResponse.getcode = lambda self: self.status
 
 try:
     from urllib.error import HTTPError as compat_HTTPError

From 46fde7caeeab13a6277aab22a0e8a29e10c30cc3 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 7 Jun 2023 14:51:50 +0100
Subject: [PATCH 665/860] [core] Update redirect handling from yt-dlp

* Thx coletdjnz: https://github.com/yt-dlp/yt-dlp/pull/7094
* add test that redirected `POST` loses its `Content-Type`
---
 test/test_http.py   | 489 +++++++++++++++++++++++++++++++++++++++-----
 youtube_dl/utils.py |  74 ++++---
 2 files changed, 484 insertions(+), 79 deletions(-)

diff --git a/test/test_http.py b/test/test_http.py
index 487a9bc77..1a65df9e0 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -8,33 +8,160 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+import gzip
+import io
+import ssl
+import tempfile
+import threading
+import zlib
+
+# avoid deprecated alias assertRaisesRegexp
+if hasattr(unittest.TestCase, 'assertRaisesRegex'):
+    unittest.TestCase.assertRaisesRegexp = unittest.TestCase.assertRaisesRegex
+
+try:
+    import brotli
+except ImportError:
+    brotli = None
+try:
+    from urllib.request import pathname2url
+except ImportError:
+    from urllib import pathname2url
+
+from youtube_dl.compat import (
+    compat_http_cookiejar_Cookie,
+    compat_http_server,
+    compat_str as str,
+    compat_urllib_error,
+    compat_urllib_HTTPError,
+    compat_urllib_parse,
+    compat_urllib_request,
+)
+
+from youtube_dl.utils import (
+    sanitized_Request,
+    urlencode_postdata,
+)
+
 from test.helper import (
+    FakeYDL,
     FakeLogger,
     http_server_port,
 )
 from youtube_dl import YoutubeDL
-from youtube_dl.compat import compat_http_server, compat_urllib_request
-import ssl
-import threading
 
 TEST_DIR = os.path.dirname(os.path.abspath(__file__))
 
 
 class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
+    protocol_version = 'HTTP/1.1'
+
+    # work-around old/new -style class inheritance
+    def super(self, meth_name, *args, **kwargs):
+        from types import MethodType
+        try:
+            super()
+            fn = lambda s, m, *a, **k: getattr(super(), m)(*a, **k)
+        except TypeError:
+            fn = lambda s, m, *a, **k: getattr(compat_http_server.BaseHTTPRequestHandler, m)(s, *a, **k)
+        self.super = MethodType(fn, self)
+        return self.super(meth_name, *args, **kwargs)
+
     def log_message(self, format, *args):
         pass
 
+    def _headers(self):
+        payload = str(self.headers).encode('utf-8')
+        self.send_response(200)
+        self.send_header('Content-Type', 'application/json')
+        self.send_header('Content-Length', str(len(payload)))
+        self.end_headers()
+        self.wfile.write(payload)
+
+    def _redirect(self):
+        self.send_response(int(self.path[len('/redirect_'):]))
+        self.send_header('Location', '/method')
+        self.send_header('Content-Length', '0')
+        self.end_headers()
+
+    def _method(self, method, payload=None):
+        self.send_response(200)
+        self.send_header('Content-Length', str(len(payload or '')))
+        self.send_header('Method', method)
+        self.end_headers()
+        if payload:
+            self.wfile.write(payload)
+
+    def _status(self, status):
+        payload = '<html>{0} NOT FOUND</html>'.format(status).encode('utf-8')
+        self.send_response(int(status))
+        self.send_header('Content-Type', 'text/html; charset=utf-8')
+        self.send_header('Content-Length', str(len(payload)))
+        self.end_headers()
+        self.wfile.write(payload)
+
+    def _read_data(self):
+        if 'Content-Length' in self.headers:
+            return self.rfile.read(int(self.headers['Content-Length']))
+
+    def _test_url(self, path, host='127.0.0.1', scheme='http', port=None):
+        return '{0}://{1}:{2}/{3}'.format(
+            scheme, host,
+            port if port is not None
+            else http_server_port(self.server), path)
+
+    def do_POST(self):
+        data = self._read_data()
+        if self.path.startswith('/redirect_'):
+            self._redirect()
+        elif self.path.startswith('/method'):
+            self._method('POST', data)
+        elif self.path.startswith('/headers'):
+            self._headers()
+        else:
+            self._status(404)
+
+    def do_HEAD(self):
+        if self.path.startswith('/redirect_'):
+            self._redirect()
+        elif self.path.startswith('/method'):
+            self._method('HEAD')
+        else:
+            self._status(404)
+
+    def do_PUT(self):
+        data = self._read_data()
+        if self.path.startswith('/redirect_'):
+            self._redirect()
+        elif self.path.startswith('/method'):
+            self._method('PUT', data)
+        else:
+            self._status(404)
+
     def do_GET(self):
+
+        def respond(payload=b'<html><video src="/vid.mp4" /></html>',
+                    payload_type='text/html; charset=utf-8',
+                    payload_encoding=None,
+                    resp_code=200):
+            self.send_response(resp_code)
+            self.send_header('Content-Type', payload_type)
+            if payload_encoding:
+                self.send_header('Content-Encoding', payload_encoding)
+            self.send_header('Content-Length', str(len(payload)))  # required for persistent connections
+            self.end_headers()
+            self.wfile.write(payload)
+
+        def gzip_compress(p):
+            buf = io.BytesIO()
+            with gzip.GzipFile(fileobj=buf, mode='wb') as f:
+                f.write(p)
+            return buf.getvalue()
+
         if self.path == '/video.html':
-            self.send_response(200)
-            self.send_header('Content-Type', 'text/html; charset=utf-8')
-            self.end_headers()
-            self.wfile.write(b'<html><video src="/vid.mp4" /></html>')
+            respond()
         elif self.path == '/vid.mp4':
-            self.send_response(200)
-            self.send_header('Content-Type', 'video/mp4')
-            self.end_headers()
-            self.wfile.write(b'\x00\x00\x00\x00\x20\x66\x74[video]')
+            respond(b'\x00\x00\x00\x00\x20\x66\x74[video]', 'video/mp4')
         elif self.path == '/302':
             if sys.version_info[0] == 3:
                 # XXX: Python 3 http server does not allow non-ASCII header values
@@ -42,60 +169,284 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
                 self.end_headers()
                 return
 
-            new_url = 'http://127.0.0.1:%d/中文.html' % http_server_port(self.server)
+            new_url = self._test_url('中文.html')
             self.send_response(302)
             self.send_header(b'Location', new_url.encode('utf-8'))
             self.end_headers()
         elif self.path == '/%E4%B8%AD%E6%96%87.html':
-            self.send_response(200)
-            self.send_header('Content-Type', 'text/html; charset=utf-8')
+            respond()
+        elif self.path == '/%c7%9f':
+            respond()
+        elif self.path.startswith('/redirect_'):
+            self._redirect()
+        elif self.path.startswith('/method'):
+            self._method('GET')
+        elif self.path.startswith('/headers'):
+            self._headers()
+        elif self.path == '/trailing_garbage':
+            payload = b'<html><video src="/vid.mp4" /></html>'
+            compressed = gzip_compress(payload) + b'trailing garbage'
+            respond(compressed, payload_encoding='gzip')
+        elif self.path == '/302-non-ascii-redirect':
+            new_url = self._test_url('中文.html')
+            # actually respond with permanent redirect
+            self.send_response(301)
+            self.send_header('Location', new_url)
+            self.send_header('Content-Length', '0')
             self.end_headers()
-            self.wfile.write(b'<html><video src="/vid.mp4" /></html>')
+        elif self.path == '/content-encoding':
+            encodings = self.headers.get('ytdl-encoding', '')
+            payload = b'<html><video src="/vid.mp4" /></html>'
+            for encoding in filter(None, (e.strip() for e in encodings.split(','))):
+                if encoding == 'br' and brotli:
+                    payload = brotli.compress(payload)
+                elif encoding == 'gzip':
+                    payload = gzip_compress(payload)
+                elif encoding == 'deflate':
+                    payload = zlib.compress(payload)
+                elif encoding == 'unsupported':
+                    payload = b'raw'
+                    break
+                else:
+                    self._status(415)
+                    return
+            respond(payload, payload_encoding=encodings)
+
         else:
-            assert False
+            self._status(404)
+
+    def send_header(self, keyword, value):
+        """
+        Forcibly allow HTTP server to send non percent-encoded non-ASCII characters in headers.
+        This is against what is defined in RFC 3986: but we need to test that we support this
+        since some sites incorrectly do this.
+        """
+        if keyword.lower() == 'connection':
+            return self.super('send_header', keyword, value)
+
+        if not hasattr(self, '_headers_buffer'):
+            self._headers_buffer = []
+
+        self._headers_buffer.append('{0}: {1}\r\n'.format(keyword, value).encode('utf-8'))
+
+    def end_headers(self):
+        if hasattr(self, '_headers_buffer'):
+            self.wfile.write(b''.join(self._headers_buffer))
+            self._headers_buffer = []
+        self.super('end_headers')
 
 
 class TestHTTP(unittest.TestCase):
     def setUp(self):
-        self.httpd = compat_http_server.HTTPServer(
+        # HTTP server
+        self.http_httpd = compat_http_server.HTTPServer(
             ('127.0.0.1', 0), HTTPTestRequestHandler)
-        self.port = http_server_port(self.httpd)
-        self.server_thread = threading.Thread(target=self.httpd.serve_forever)
-        self.server_thread.daemon = True
-        self.server_thread.start()
+        self.http_port = http_server_port(self.http_httpd)
 
-    def test_unicode_path_redirection(self):
-        # XXX: Python 3 http server does not allow non-ASCII header values
-        if sys.version_info[0] == 3:
-            return
+        self.http_server_thread = threading.Thread(target=self.http_httpd.serve_forever)
+        self.http_server_thread.daemon = True
+        self.http_server_thread.start()
 
-        ydl = YoutubeDL({'logger': FakeLogger()})
-        r = ydl.extract_info('http://127.0.0.1:%d/302' % self.port)
-        self.assertEqual(r['entries'][0]['url'], 'http://127.0.0.1:%d/vid.mp4' % self.port)
+        try:
+            from http.server import ThreadingHTTPServer
+        except ImportError:
+            try:
+                from socketserver import ThreadingMixIn
+            except ImportError:
+                from SocketServer import ThreadingMixIn
 
+            class ThreadingHTTPServer(ThreadingMixIn, compat_http_server.HTTPServer):
+                pass
 
-class TestHTTPS(unittest.TestCase):
-    def setUp(self):
+        # HTTPS server
         certfn = os.path.join(TEST_DIR, 'testcert.pem')
-        self.httpd = compat_http_server.HTTPServer(
+        self.https_httpd = ThreadingHTTPServer(
             ('127.0.0.1', 0), HTTPTestRequestHandler)
-        self.httpd.socket = ssl.wrap_socket(
-            self.httpd.socket, certfile=certfn, server_side=True)
-        self.port = http_server_port(self.httpd)
-        self.server_thread = threading.Thread(target=self.httpd.serve_forever)
-        self.server_thread.daemon = True
-        self.server_thread.start()
+        try:
+            sslctx = ssl.SSLContext(ssl.PROTOCOL_SSLv23)
+            sslctx.verify_mode = ssl.CERT_NONE
+            sslctx.check_hostname = False
+            sslctx.load_cert_chain(certfn, None)
+            self.https_httpd.socket = sslctx.wrap_socket(
+                self.https_httpd.socket, server_side=True)
+        except AttributeError:
+            self.https_httpd.socket = ssl.wrap_socket(
+                self.https_httpd.socket, certfile=certfn, server_side=True)
+
+        self.https_port = http_server_port(self.https_httpd)
+        self.https_server_thread = threading.Thread(target=self.https_httpd.serve_forever)
+        self.https_server_thread.daemon = True
+        self.https_server_thread.start()
+
+    def tearDown(self):
+
+        def closer(svr):
+            def _closer():
+                svr.shutdown()
+                svr.server_close()
+            return _closer
+
+        shutdown_thread = threading.Thread(target=closer(self.http_httpd))
+        shutdown_thread.start()
+        self.http_server_thread.join(2.0)
+
+        shutdown_thread = threading.Thread(target=closer(self.https_httpd))
+        shutdown_thread.start()
+        self.https_server_thread.join(2.0)
+
+    def _test_url(self, path, host='127.0.0.1', scheme='http', port=None):
+        return '{0}://{1}:{2}/{3}'.format(
+            scheme, host,
+            port if port is not None
+            else self.https_port if scheme == 'https'
+            else self.http_port, path)
 
     def test_nocheckcertificate(self):
-        if sys.version_info >= (2, 7, 9):  # No certificate checking anyways
-            ydl = YoutubeDL({'logger': FakeLogger()})
-            self.assertRaises(
-                Exception,
-                ydl.extract_info, 'https://127.0.0.1:%d/video.html' % self.port)
+        with FakeYDL({'logger': FakeLogger()}) as ydl:
+            with self.assertRaises(compat_urllib_error.URLError):
+                ydl.urlopen(sanitized_Request(self._test_url('headers', scheme='https')))
 
-        ydl = YoutubeDL({'logger': FakeLogger(), 'nocheckcertificate': True})
-        r = ydl.extract_info('https://127.0.0.1:%d/video.html' % self.port)
-        self.assertEqual(r['entries'][0]['url'], 'https://127.0.0.1:%d/vid.mp4' % self.port)
+        with FakeYDL({'logger': FakeLogger(), 'nocheckcertificate': True}) as ydl:
+            r = ydl.urlopen(sanitized_Request(self._test_url('headers', scheme='https')))
+            self.assertEqual(r.getcode(), 200)
+            r.close()
+
+    def test_percent_encode(self):
+        with FakeYDL() as ydl:
+            # Unicode characters should be encoded with uppercase percent-encoding
+            res = ydl.urlopen(sanitized_Request(self._test_url('中文.html')))
+            self.assertEqual(res.getcode(), 200)
+            res.close()
+            # don't normalize existing percent encodings
+            res = ydl.urlopen(sanitized_Request(self._test_url('%c7%9f')))
+            self.assertEqual(res.getcode(), 200)
+            res.close()
+
+    def test_unicode_path_redirection(self):
+        with FakeYDL() as ydl:
+            r = ydl.urlopen(sanitized_Request(self._test_url('302-non-ascii-redirect')))
+            self.assertEqual(r.url, self._test_url('%E4%B8%AD%E6%96%87.html'))
+            r.close()
+
+    def test_redirect(self):
+        with FakeYDL() as ydl:
+            def do_req(redirect_status, method, check_no_content=False):
+                data = b'testdata' if method in ('POST', 'PUT') else None
+                res = ydl.urlopen(sanitized_Request(
+                    self._test_url('redirect_{0}'.format(redirect_status)),
+                    method=method, data=data))
+                if check_no_content:
+                    self.assertNotIn('Content-Type', res.headers)
+                return res.read().decode('utf-8'), res.headers.get('method', '')
+            # A 303 must either use GET or HEAD for subsequent request
+            self.assertEqual(do_req(303, 'POST'), ('', 'GET'))
+            self.assertEqual(do_req(303, 'HEAD'), ('', 'HEAD'))
+
+            self.assertEqual(do_req(303, 'PUT'), ('', 'GET'))
+
+            # 301 and 302 turn POST only into a GET, with no Content-Type
+            self.assertEqual(do_req(301, 'POST', True), ('', 'GET'))
+            self.assertEqual(do_req(301, 'HEAD'), ('', 'HEAD'))
+            self.assertEqual(do_req(302, 'POST', True), ('', 'GET'))
+            self.assertEqual(do_req(302, 'HEAD'), ('', 'HEAD'))
+
+            self.assertEqual(do_req(301, 'PUT'), ('testdata', 'PUT'))
+            self.assertEqual(do_req(302, 'PUT'), ('testdata', 'PUT'))
+
+            # 307 and 308 should not change method
+            for m in ('POST', 'PUT'):
+                self.assertEqual(do_req(307, m), ('testdata', m))
+                self.assertEqual(do_req(308, m), ('testdata', m))
+
+            self.assertEqual(do_req(307, 'HEAD'), ('', 'HEAD'))
+            self.assertEqual(do_req(308, 'HEAD'), ('', 'HEAD'))
+
+            # These should not redirect and instead raise an HTTPError
+            for code in (300, 304, 305, 306):
+                with self.assertRaises(compat_urllib_HTTPError):
+                    do_req(code, 'GET')
+
+    def test_content_type(self):
+        # https://github.com/yt-dlp/yt-dlp/commit/379a4f161d4ad3e40932dcf5aca6e6fb9715ab28
+        with FakeYDL({'nocheckcertificate': True}) as ydl:
+            # method should be auto-detected as POST
+            r = sanitized_Request(self._test_url('headers', scheme='https'), data=urlencode_postdata({'test': 'test'}))
+
+            headers = ydl.urlopen(r).read().decode('utf-8')
+            self.assertIn('Content-Type: application/x-www-form-urlencoded', headers)
+
+            # test http
+            r = sanitized_Request(self._test_url('headers'), data=urlencode_postdata({'test': 'test'}))
+            headers = ydl.urlopen(r).read().decode('utf-8')
+            self.assertIn('Content-Type: application/x-www-form-urlencoded', headers)
+
+    def test_cookiejar(self):
+        with FakeYDL() as ydl:
+            ydl.cookiejar.set_cookie(compat_http_cookiejar_Cookie(
+                0, 'test', 'ytdl', None, False, '127.0.0.1', True,
+                False, '/headers', True, False, None, False, None, None, {}))
+            data = ydl.urlopen(sanitized_Request(self._test_url('headers'))).read()
+            self.assertIn(b'Cookie: test=ytdl', data)
+
+    def test_no_compression_compat_header(self):
+        with FakeYDL() as ydl:
+            data = ydl.urlopen(
+                sanitized_Request(
+                    self._test_url('headers'),
+                    headers={'Youtubedl-no-compression': True})).read()
+            self.assertIn(b'Accept-Encoding: identity', data)
+            self.assertNotIn(b'youtubedl-no-compression', data.lower())
+
+    def test_gzip_trailing_garbage(self):
+        # https://github.com/ytdl-org/youtube-dl/commit/aa3e950764337ef9800c936f4de89b31c00dfcf5
+        # https://github.com/ytdl-org/youtube-dl/commit/6f2ec15cee79d35dba065677cad9da7491ec6e6f
+        with FakeYDL() as ydl:
+            data = ydl.urlopen(sanitized_Request(self._test_url('trailing_garbage'))).read().decode('utf-8')
+            self.assertEqual(data, '<html><video src="/vid.mp4" /></html>')
+
+    def __test_compression(self, encoding):
+        with FakeYDL() as ydl:
+            res = ydl.urlopen(
+                sanitized_Request(
+                    self._test_url('content-encoding'),
+                    headers={'ytdl-encoding': encoding}))
+            self.assertEqual(res.headers.get('Content-Encoding'), encoding)
+            self.assertEqual(res.read(), b'<html><video src="/vid.mp4" /></html>')
+
+    @unittest.skipUnless(brotli, 'brotli support is not installed')
+    @unittest.expectedFailure
+    def test_brotli(self):
+        self.__test_compression('br')
+
+    @unittest.expectedFailure
+    def test_deflate(self):
+        self.__test_compression('deflate')
+
+    @unittest.expectedFailure
+    def test_gzip(self):
+        self.__test_compression('gzip')
+
+    @unittest.expectedFailure  # not yet implemented
+    def test_multiple_encodings(self):
+        # https://www.rfc-editor.org/rfc/rfc9110.html#section-8.4
+        with FakeYDL() as ydl:
+            for pair in ('gzip,deflate', 'deflate, gzip', 'gzip, gzip', 'deflate, deflate'):
+                res = ydl.urlopen(
+                    sanitized_Request(
+                        self._test_url('content-encoding'),
+                        headers={'ytdl-encoding': pair}))
+                self.assertEqual(res.headers.get('Content-Encoding'), pair)
+                self.assertEqual(res.read(), b'<html><video src="/vid.mp4" /></html>')
+
+    def test_unsupported_encoding(self):
+        # it should return the raw content
+        with FakeYDL() as ydl:
+            res = ydl.urlopen(
+                sanitized_Request(
+                    self._test_url('content-encoding'),
+                    headers={'ytdl-encoding': 'unsupported'}))
+            self.assertEqual(res.headers.get('Content-Encoding'), 'unsupported')
+            self.assertEqual(res.read(), b'raw')
 
 
 def _build_proxy_handler(name):
@@ -109,7 +460,7 @@ def _build_proxy_handler(name):
             self.send_response(200)
             self.send_header('Content-Type', 'text/plain; charset=utf-8')
             self.end_headers()
-            self.wfile.write('{self.proxy_name}: {self.path}'.format(self=self).encode('utf-8'))
+            self.wfile.write('{0}: {1}'.format(self.proxy_name, self.path).encode('utf-8'))
     return HTTPTestRequestHandler
 
 
@@ -129,10 +480,30 @@ class TestProxy(unittest.TestCase):
         self.geo_proxy_thread.daemon = True
         self.geo_proxy_thread.start()
 
+    def tearDown(self):
+
+        def closer(svr):
+            def _closer():
+                svr.shutdown()
+                svr.server_close()
+            return _closer
+
+        shutdown_thread = threading.Thread(target=closer(self.proxy))
+        shutdown_thread.start()
+        self.proxy_thread.join(2.0)
+
+        shutdown_thread = threading.Thread(target=closer(self.geo_proxy))
+        shutdown_thread.start()
+        self.geo_proxy_thread.join(2.0)
+
+    def _test_proxy(self, host='127.0.0.1', port=None):
+        return '{0}:{1}'.format(
+            host, port if port is not None else self.port)
+
     def test_proxy(self):
-        geo_proxy = '127.0.0.1:{0}'.format(self.geo_port)
+        geo_proxy = self._test_proxy(port=self.geo_port)
         ydl = YoutubeDL({
-            'proxy': '127.0.0.1:{0}'.format(self.port),
+            'proxy': self._test_proxy(),
             'geo_verification_proxy': geo_proxy,
         })
         url = 'http://foo.com/bar'
@@ -146,7 +517,7 @@ class TestProxy(unittest.TestCase):
 
     def test_proxy_with_idn(self):
         ydl = YoutubeDL({
-            'proxy': '127.0.0.1:{0}'.format(self.port),
+            'proxy': self._test_proxy(),
         })
         url = 'http://中文.tw/'
         response = ydl.urlopen(url).read().decode('utf-8')
@@ -154,5 +525,25 @@ class TestProxy(unittest.TestCase):
         self.assertEqual(response, 'normal: http://xn--fiq228c.tw/')
 
 
+class TestFileURL(unittest.TestCase):
+    # See https://github.com/ytdl-org/youtube-dl/issues/8227
+    def test_file_urls(self):
+        tf = tempfile.NamedTemporaryFile(delete=False)
+        tf.write(b'foobar')
+        tf.close()
+        url = compat_urllib_parse.urljoin('file://', pathname2url(tf.name))
+        with FakeYDL() as ydl:
+            self.assertRaisesRegexp(
+                compat_urllib_error.URLError, 'file:// scheme is explicitly disabled in youtube-dl for security reasons', ydl.urlopen, url)
+        # not yet implemented
+        """
+        with FakeYDL({'enable_file_urls': True}) as ydl:
+            res = ydl.urlopen(url)
+            self.assertEqual(res.read(), b'foobar')
+            res.close()
+        """
+        os.unlink(tf.name)
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index dbdbe5f59..58c710b08 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -41,7 +41,6 @@ import zlib
 from .compat import (
     compat_HTMLParseError,
     compat_HTMLParser,
-    compat_HTTPError,
     compat_basestring,
     compat_casefold,
     compat_chr,
@@ -64,6 +63,7 @@ from .compat import (
     compat_struct_pack,
     compat_struct_unpack,
     compat_urllib_error,
+    compat_urllib_HTTPError,
     compat_urllib_parse,
     compat_urllib_parse_parse_qs as compat_parse_qs,
     compat_urllib_parse_urlencode,
@@ -2614,7 +2614,8 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
 
     Part of this code was copied from:
 
-    http://techknack.net/python-urllib2-handlers/
+    http://techknack.net/python-urllib2-handlers/, archived at
+    https://web.archive.org/web/20130527205558/http://techknack.net/python-urllib2-handlers/
 
     Andrew Rowls, the author of that code, agreed to release it to the
     public domain.
@@ -2672,7 +2673,9 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
             req._Request__original = req._Request__original.partition('#')[0]
             req._Request__r_type = req._Request__r_type.partition('#')[0]
 
-        return req
+        # Use the totally undocumented AbstractHTTPHandler per
+        # https://github.com/yt-dlp/yt-dlp/pull/4158
+        return compat_urllib_request.AbstractHTTPHandler.do_request_(self, req)
 
     def http_response(self, req, resp):
         old_resp = resp
@@ -2683,7 +2686,7 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
             try:
                 uncompressed = io.BytesIO(gz.read())
             except IOError as original_ioerror:
-                # There may be junk add the end of the file
+                # There may be junk at the end of the file
                 # See http://stackoverflow.com/q/4928560/35070 for details
                 for i in range(1, 1024):
                     try:
@@ -2710,9 +2713,8 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
             if location:
                 # As of RFC 2616 default charset is iso-8859-1 that is respected by python 3
                 if sys.version_info >= (3, 0):
-                    location = location.encode('iso-8859-1').decode('utf-8')
-                else:
-                    location = location.decode('utf-8')
+                    location = location.encode('iso-8859-1')
+                location = location.decode('utf-8')
                 location_escaped = escape_url(location)
                 if location != location_escaped:
                     del resp.headers['Location']
@@ -2940,17 +2942,16 @@ class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
 
     The code is based on HTTPRedirectHandler implementation from CPython [1].
 
-    This redirect handler solves two issues:
-     - ensures redirect URL is always unicode under python 2
-     - introduces support for experimental HTTP response status code
-       308 Permanent Redirect [2] used by some sites [3]
+    This redirect handler fixes and improves the logic to better align with RFC7261
+    and what browsers tend to do [2][3]
 
     1. https://github.com/python/cpython/blob/master/Lib/urllib/request.py
-    2. https://developer.mozilla.org/en-US/docs/Web/HTTP/Status/308
-    3. https://github.com/ytdl-org/youtube-dl/issues/28768
+    2. https://datatracker.ietf.org/doc/html/rfc7231
+    3. https://github.com/python/cpython/issues/91306
     """
 
-    http_error_301 = http_error_303 = http_error_307 = http_error_308 = compat_urllib_request.HTTPRedirectHandler.http_error_302
+    # Supply possibly missing alias
+    http_error_308 = compat_urllib_request.HTTPRedirectHandler.http_error_302
 
     def redirect_request(self, req, fp, code, msg, headers, newurl):
         """Return a Request or None in response to a redirect.
@@ -2962,19 +2963,16 @@ class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
         else should try to handle this url.  Return None if you can't
         but another Handler might.
         """
-        m = req.get_method()
-        if (not (code in (301, 302, 303, 307, 308) and m in ("GET", "HEAD")
-                 or code in (301, 302, 303) and m == "POST")):
-            raise compat_HTTPError(req.full_url, code, msg, headers, fp)
-        # Strictly (according to RFC 2616), 301 or 302 in response to
-        # a POST MUST NOT cause a redirection without confirmation
-        # from the user (of urllib.request, in this case).  In practice,
-        # essentially all clients do redirect in this case, so we do
-        # the same.
+        if code not in (301, 302, 303, 307, 308):
+            raise compat_urllib_HTTPError(req.full_url, code, msg, headers, fp)
+
+        new_method = req.get_method()
+        new_data = req.data
+        remove_headers = []
 
         # On python 2 urlh.geturl() may sometimes return redirect URL
-        # as byte string instead of unicode. This workaround allows
-        # to force it always return unicode.
+        # as a byte string instead of unicode. This workaround forces
+        # it to return unicode.
         if sys.version_info[0] < 3:
             newurl = compat_str(newurl)
 
@@ -2983,13 +2981,29 @@ class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
         # but it is kept for compatibility with other callers.
         newurl = newurl.replace(' ', '%20')
 
-        CONTENT_HEADERS = ("content-length", "content-type")
+        # A 303 must either use GET or HEAD for subsequent request
+        # https://datatracker.ietf.org/doc/html/rfc7231#section-6.4.4
+        if code == 303 and req.get_method() != 'HEAD':
+            new_method = 'GET'
+        # 301 and 302 redirects are commonly turned into a GET from a POST
+        # for subsequent requests by browsers, so we'll do the same.
+        # https://datatracker.ietf.org/doc/html/rfc7231#section-6.4.2
+        # https://datatracker.ietf.org/doc/html/rfc7231#section-6.4.3
+        elif code in (301, 302) and req.get_method() == 'POST':
+            new_method = 'GET'
+
+        # only remove payload if method changed (e.g. POST to GET)
+        if new_method != req.get_method():
+            new_data = None
+            remove_headers.extend(['Content-Length', 'Content-Type'])
+
         # NB: don't use dict comprehension for python 2.6 compatibility
-        newheaders = dict((k, v) for k, v in req.headers.items()
-                          if k.lower() not in CONTENT_HEADERS)
+        new_headers = dict((k, v) for k, v in req.header_items()
+                           if k.lower() not in remove_headers)
+
         return compat_urllib_request.Request(
-            newurl, headers=newheaders, origin_req_host=req.origin_req_host,
-            unverifiable=True)
+            newurl, headers=new_headers, origin_req_host=req.origin_req_host,
+            unverifiable=True, method=new_method, data=new_data)
 
 
 def extract_timezone(date_str):

From b383be98874d4dded67ee8a679fae30340722709 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 7 Jun 2023 19:38:54 +0100
Subject: [PATCH 666/860] [core] Remove `Cookie` header on redirect to prevent
 leaks

Adated from yt-dlp/yt-dlp-ghsa-v8mc-9377-rwjj/pull/1/commits/101caac
Thx coletdjnz
---
 test/test_http.py   | 32 ++++++++++++++++++++++++++++++--
 youtube_dl/utils.py |  8 ++++++--
 2 files changed, 36 insertions(+), 4 deletions(-)

diff --git a/test/test_http.py b/test/test_http.py
index 1a65df9e0..cd180b51f 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -183,6 +183,11 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
             self._method('GET')
         elif self.path.startswith('/headers'):
             self._headers()
+        elif self.path.startswith('/308-to-headers'):
+            self.send_response(308)
+            self.send_header('Location', '/headers')
+            self.send_header('Content-Length', '0')
+            self.end_headers()
         elif self.path == '/trailing_garbage':
             payload = b'<html><video src="/vid.mp4" /></html>'
             compressed = gzip_compress(payload) + b'trailing garbage'
@@ -385,8 +390,31 @@ class TestHTTP(unittest.TestCase):
             ydl.cookiejar.set_cookie(compat_http_cookiejar_Cookie(
                 0, 'test', 'ytdl', None, False, '127.0.0.1', True,
                 False, '/headers', True, False, None, False, None, None, {}))
-            data = ydl.urlopen(sanitized_Request(self._test_url('headers'))).read()
-            self.assertIn(b'Cookie: test=ytdl', data)
+            data = ydl.urlopen(sanitized_Request(
+                self._test_url('headers'))).read().decode('utf-8')
+            self.assertIn('Cookie: test=ytdl', data)
+
+    def test_passed_cookie_header(self):
+        # We should accept a Cookie header being passed as in normal headers and handle it appropriately.
+        with FakeYDL() as ydl:
+            # Specified Cookie header should be used
+            res = ydl.urlopen(sanitized_Request(
+                self._test_url('headers'), headers={'Cookie': 'test=test'})).read().decode('utf-8')
+            self.assertIn('Cookie: test=test', res)
+
+            # Specified Cookie header should be removed on any redirect
+            res = ydl.urlopen(sanitized_Request(
+                self._test_url('308-to-headers'), headers={'Cookie': 'test=test'})).read().decode('utf-8')
+            self.assertNotIn('Cookie: test=test', res)
+
+            # Specified Cookie header should override global cookiejar for that request
+            ydl.cookiejar.set_cookie(compat_http_cookiejar_Cookie(
+                0, 'test', 'ytdlp', None, False, '127.0.0.1', True,
+                False, '/headers', True, False, None, False, None, None, {}))
+            data = ydl.urlopen(sanitized_Request(
+                self._test_url('headers'), headers={'Cookie': 'test=test'})).read().decode('utf-8')
+            self.assertNotIn('Cookie: test=ytdlp', data)
+            self.assertIn('Cookie: test=test', data)
 
     def test_no_compression_compat_header(self):
         with FakeYDL() as ydl:
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 58c710b08..c21cd3687 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2968,7 +2968,6 @@ class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
 
         new_method = req.get_method()
         new_data = req.data
-        remove_headers = []
 
         # On python 2 urlh.geturl() may sometimes return redirect URL
         # as a byte string instead of unicode. This workaround forces
@@ -2981,6 +2980,11 @@ class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
         # but it is kept for compatibility with other callers.
         newurl = newurl.replace(' ', '%20')
 
+        # Technically the Cookie header should be in unredirected_hdrs;
+        # however in practice some may set it in normal headers anyway.
+        # We will remove it here to prevent any leaks.
+        remove_headers = ['Cookie']
+
         # A 303 must either use GET or HEAD for subsequent request
         # https://datatracker.ietf.org/doc/html/rfc7231#section-6.4.4
         if code == 303 and req.get_method() != 'HEAD':
@@ -2999,7 +3003,7 @@ class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
 
         # NB: don't use dict comprehension for python 2.6 compatibility
         new_headers = dict((k, v) for k, v in req.header_items()
-                           if k.lower() not in remove_headers)
+                           if k.title() not in remove_headers)
 
         return compat_urllib_request.Request(
             newurl, headers=new_headers, origin_req_host=req.origin_req_host,

From 3801d36416d6e3e6031dc4fcac01891ce7ddb55b Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Tue, 4 Jul 2023 14:03:39 -0500
Subject: [PATCH 667/860] [utils] `YoutubeDLCookieJar`: Add `get_cookie_header`
 and `get_cookies_for_url` methods

---
 youtube_dl/utils.py | 13 +++++++++++++
 1 file changed, 13 insertions(+)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c21cd3687..ac6c81465 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2912,6 +2912,19 @@ class YoutubeDLCookieJar(compat_cookiejar.MozillaCookieJar):
                 cookie.expires = None
                 cookie.discard = True
 
+    def get_cookie_header(self, url):
+        """Generate a Cookie HTTP header for a given url"""
+        cookie_req = sanitized_Request(url)
+        self.add_cookie_header(cookie_req)
+        return cookie_req.get_header('Cookie')
+
+    def get_cookies_for_url(self, url):
+        """Generate a list of Cookie objects for a given url"""
+        # Policy `_now` attribute must be set before calling `_cookies_for_request`
+        # Ref: https://github.com/python/cpython/blob/3.7/Lib/http/cookiejar.py#L1360
+        self._policy._now = self._now = int(time.time())
+        return self._cookies_for_request(sanitized_Request(url))
+
 
 class YoutubeDLCookieProcessor(compat_urllib_request.HTTPCookieProcessor):
     def __init__(self, cookiejar=None):

From 8334ec961b802ad7ef8571b776c5fc727206dc9b Mon Sep 17 00:00:00 2001
From: Simon Sawicki <contact@grub4k.xyz>
Date: Tue, 4 Jul 2023 21:41:04 +0200
Subject: [PATCH 668/860] [core] Process header cookies on loading

---
 test/test_YoutubeDL.py          | 185 +++++++++++++++++++++++++++++++-
 test/test_YoutubeDLCookieJar.py |  14 +++
 youtube_dl/YoutubeDL.py         | 182 ++++++++++++++++++++++++++-----
 youtube_dl/downloader/common.py |   9 ++
 4 files changed, 357 insertions(+), 33 deletions(-)

diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 60780b8a7..6cf555827 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -10,14 +10,30 @@ import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import copy
+import json
 
-from test.helper import FakeYDL, assertRegexpMatches
+from test.helper import (
+    FakeYDL,
+    assertRegexpMatches,
+    try_rm,
+)
 from youtube_dl import YoutubeDL
-from youtube_dl.compat import compat_str, compat_urllib_error
+from youtube_dl.compat import (
+    compat_http_cookiejar_Cookie,
+    compat_http_cookies_SimpleCookie,
+    compat_kwargs,
+    compat_str,
+    compat_urllib_error,
+)
+
 from youtube_dl.extractor import YoutubeIE
 from youtube_dl.extractor.common import InfoExtractor
 from youtube_dl.postprocessor.common import PostProcessor
-from youtube_dl.utils import ExtractorError, match_filter_func
+from youtube_dl.utils import (
+    ExtractorError,
+    match_filter_func,
+    traverse_obj,
+)
 
 TEST_URL = 'http://localhost/sample.mp4'
 
@@ -29,11 +45,14 @@ class YDL(FakeYDL):
         self.msgs = []
 
     def process_info(self, info_dict):
-        self.downloaded_info_dicts.append(info_dict)
+        self.downloaded_info_dicts.append(info_dict.copy())
 
     def to_screen(self, msg):
         self.msgs.append(msg)
 
+    def dl(self, *args, **kwargs):
+        assert False, 'Downloader must not be invoked for test_YoutubeDL'
+
 
 def _make_result(formats, **kwargs):
     res = {
@@ -42,8 +61,9 @@ def _make_result(formats, **kwargs):
         'title': 'testttitle',
         'extractor': 'testex',
         'extractor_key': 'TestEx',
+        'webpage_url': 'http://example.com/watch?v=shenanigans',
     }
-    res.update(**kwargs)
+    res.update(**compat_kwargs(kwargs))
     return res
 
 
@@ -1011,5 +1031,160 @@ class TestYoutubeDL(unittest.TestCase):
         self.assertEqual(out_info['release_date'], '20210930')
 
 
+class TestYoutubeDLCookies(unittest.TestCase):
+
+    @staticmethod
+    def encode_cookie(cookie):
+        if not isinstance(cookie, dict):
+            cookie = vars(cookie)
+        for name, value in cookie.items():
+            yield name, compat_str(value)
+
+    @classmethod
+    def comparable_cookies(cls, cookies):
+        # Work around cookiejar cookies not being unicode strings
+        return sorted(map(tuple, map(sorted, map(cls.encode_cookie, cookies))))
+
+    def assertSameCookies(self, c1, c2, msg=None):
+        return self.assertEqual(
+            *map(self.comparable_cookies, (c1, c2)),
+            msg=msg)
+
+    def assertSameCookieStrings(self, c1, c2, msg=None):
+        return self.assertSameCookies(
+            *map(lambda c: compat_http_cookies_SimpleCookie(c).values(), (c1, c2)),
+            msg=msg)
+
+    def test_header_cookies(self):
+
+        ydl = FakeYDL()
+        ydl.report_warning = lambda *_, **__: None
+
+        def cookie(name, value, version=None, domain='', path='', secure=False, expires=None):
+            return compat_http_cookiejar_Cookie(
+                version or 0, name, value, None, False,
+                domain, bool(domain), bool(domain), path, bool(path),
+                secure, expires, False, None, None, rest={})
+
+        test_url, test_domain = (t % ('yt.dl',) for t in ('https://%s/test', '.%s'))
+
+        def test(encoded_cookies, cookies, headers=False, round_trip=None, error_re=None):
+            def _test():
+                ydl.cookiejar.clear()
+                ydl._load_cookies(encoded_cookies, autoscope=headers)
+                if headers:
+                    ydl._apply_header_cookies(test_url)
+                data = {'url': test_url}
+                ydl._calc_headers(data)
+                self.assertSameCookies(
+                    cookies, ydl.cookiejar,
+                    'Extracted cookiejar.Cookie is not the same')
+                if not headers:
+                    self.assertSameCookieStrings(
+                        data.get('cookies'), round_trip or encoded_cookies,
+                        msg='Cookie is not the same as round trip')
+                ydl.__dict__['_YoutubeDL__header_cookies'] = []
+
+            try:
+                _test()
+            except AssertionError:
+                raise
+            except Exception as e:
+                if not error_re:
+                    raise
+                assertRegexpMatches(self, e.args[0], error_re.join(('.*',) * 2))
+
+        test('test=value; Domain=' + test_domain, [cookie('test', 'value', domain=test_domain)])
+        test('test=value', [cookie('test', 'value')], error_re='Unscoped cookies are not allowed')
+        test('cookie1=value1; Domain={0}; Path=/test; cookie2=value2; Domain={0}; Path=/'.format(test_domain), [
+            cookie('cookie1', 'value1', domain=test_domain, path='/test'),
+            cookie('cookie2', 'value2', domain=test_domain, path='/')])
+        cookie_kw = compat_kwargs(
+            {'domain': test_domain, 'path': '/test', 'secure': True, 'expires': '9999999999', })
+        test('test=value; Domain={domain}; Path={path}; Secure; Expires={expires}'.format(**cookie_kw), [
+            cookie('test', 'value', **cookie_kw)])
+        test('test="value; "; path=/test; domain=' + test_domain, [
+            cookie('test', 'value; ', domain=test_domain, path='/test')],
+            round_trip='test="value\\073 "; Domain={0}; Path=/test'.format(test_domain))
+        test('name=; Domain=' + test_domain, [cookie('name', '', domain=test_domain)],
+             round_trip='name=""; Domain=' + test_domain)
+        test('test=value', [cookie('test', 'value', domain=test_domain)], headers=True)
+        test('cookie1=value; Domain={0}; cookie2=value'.format(test_domain), [],
+             headers=True, error_re='Invalid syntax')
+        ydl.report_warning = ydl.report_error
+        test('test=value', [], headers=True, error_re='Passing cookies as a header is a potential security risk')
+
+    def test_infojson_cookies(self):
+        TEST_FILE = 'test_infojson_cookies.info.json'
+        TEST_URL = 'https://example.com/example.mp4'
+        COOKIES = 'a=b; Domain=.example.com; c=d; Domain=.example.com'
+        COOKIE_HEADER = {'Cookie': 'a=b; c=d'}
+
+        ydl = FakeYDL()
+        ydl.process_info = lambda x: ydl._write_info_json('test', x, TEST_FILE)
+
+        def make_info(info_header_cookies=False, fmts_header_cookies=False, cookies_field=False):
+            fmt = {'url': TEST_URL}
+            if fmts_header_cookies:
+                fmt['http_headers'] = COOKIE_HEADER
+            if cookies_field:
+                fmt['cookies'] = COOKIES
+            return _make_result([fmt], http_headers=COOKIE_HEADER if info_header_cookies else None)
+
+        def test(initial_info, note):
+
+            def failure_msg(why):
+                return ' when '.join((why, note))
+
+            result = {}
+            result['processed'] = ydl.process_ie_result(initial_info)
+            self.assertTrue(ydl.cookiejar.get_cookies_for_url(TEST_URL),
+                            msg=failure_msg('No cookies set in cookiejar after initial process'))
+            ydl.cookiejar.clear()
+            with open(TEST_FILE) as infojson:
+                result['loaded'] = ydl.sanitize_info(json.load(infojson), True)
+            result['final'] = ydl.process_ie_result(result['loaded'].copy(), download=False)
+            self.assertTrue(ydl.cookiejar.get_cookies_for_url(TEST_URL),
+                            msg=failure_msg('No cookies set in cookiejar after final process'))
+            ydl.cookiejar.clear()
+            for key in ('processed', 'loaded', 'final'):
+                info = result[key]
+                self.assertIsNone(
+                    traverse_obj(info, ((None, ('formats', 0)), 'http_headers', 'Cookie'), casesense=False, get_all=False),
+                    msg=failure_msg('Cookie header not removed in {0} result'.format(key)))
+                self.assertSameCookieStrings(
+                    traverse_obj(info, ((None, ('formats', 0)), 'cookies'), get_all=False), COOKIES,
+                    msg=failure_msg('No cookies field found in {0} result'.format(key)))
+
+        test({'url': TEST_URL, 'http_headers': COOKIE_HEADER, 'id': '1', 'title': 'x'}, 'no formats field')
+        test(make_info(info_header_cookies=True), 'info_dict header cokies')
+        test(make_info(fmts_header_cookies=True), 'format header cookies')
+        test(make_info(info_header_cookies=True, fmts_header_cookies=True), 'info_dict and format header cookies')
+        test(make_info(info_header_cookies=True, fmts_header_cookies=True, cookies_field=True), 'all cookies fields')
+        test(make_info(cookies_field=True), 'cookies format field')
+        test({'url': TEST_URL, 'cookies': COOKIES, 'id': '1', 'title': 'x'}, 'info_dict cookies field only')
+
+        try_rm(TEST_FILE)
+
+    def test_add_headers_cookie(self):
+        def check_for_cookie_header(result):
+            return traverse_obj(result, ((None, ('formats', 0)), 'http_headers', 'Cookie'), casesense=False, get_all=False)
+
+        ydl = FakeYDL({'http_headers': {'Cookie': 'a=b'}})
+        ydl._apply_header_cookies(_make_result([])['webpage_url'])  # Scope to input webpage URL: .example.com
+
+        fmt = {'url': 'https://example.com/video.mp4'}
+        result = ydl.process_ie_result(_make_result([fmt]), download=False)
+        self.assertIsNone(check_for_cookie_header(result), msg='http_headers cookies in result info_dict')
+        self.assertEqual(result.get('cookies'), 'a=b; Domain=.example.com', msg='No cookies were set in cookies field')
+        self.assertIn('a=b', ydl.cookiejar.get_cookie_header(fmt['url']), msg='No cookies were set in cookiejar')
+
+        fmt = {'url': 'https://wrong.com/video.mp4'}
+        result = ydl.process_ie_result(_make_result([fmt]), download=False)
+        self.assertIsNone(check_for_cookie_header(result), msg='http_headers cookies for wrong domain')
+        self.assertFalse(result.get('cookies'), msg='Cookies set in cookies field for wrong domain')
+        self.assertFalse(ydl.cookiejar.get_cookie_header(fmt['url']), msg='Cookies set in cookiejar for wrong domain')
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_YoutubeDLCookieJar.py b/test/test_YoutubeDLCookieJar.py
index 05f48bd74..4f9dd71ae 100644
--- a/test/test_YoutubeDLCookieJar.py
+++ b/test/test_YoutubeDLCookieJar.py
@@ -46,6 +46,20 @@ class TestYoutubeDLCookieJar(unittest.TestCase):
         # will be ignored
         self.assertFalse(cookiejar._cookies)
 
+    def test_get_cookie_header(self):
+        cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/httponly_cookies.txt')
+        cookiejar.load(ignore_discard=True, ignore_expires=True)
+        header = cookiejar.get_cookie_header('https://www.foobar.foobar')
+        self.assertIn('HTTPONLY_COOKIE', header)
+
+    def test_get_cookies_for_url(self):
+        cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/session_cookies.txt')
+        cookiejar.load(ignore_discard=True, ignore_expires=True)
+        cookies = cookiejar.get_cookies_for_url('https://www.foobar.foobar/')
+        self.assertEqual(len(cookies), 2)
+        cookies = cookiejar.get_cookies_for_url('https://foobar.foobar/')
+        self.assertFalse(cookies)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 1435754c2..98d080f43 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -5,6 +5,7 @@ from __future__ import absolute_import, unicode_literals
 
 import collections
 import contextlib
+import copy
 import datetime
 import errno
 import fileinput
@@ -34,10 +35,12 @@ from string import ascii_letters
 
 from .compat import (
     compat_basestring,
-    compat_cookiejar,
+    compat_collections_chain_map as ChainMap,
     compat_filter as filter,
     compat_get_terminal_size,
     compat_http_client,
+    compat_http_cookiejar_Cookie,
+    compat_http_cookies_SimpleCookie,
     compat_integer_types,
     compat_kwargs,
     compat_map as map,
@@ -53,6 +56,7 @@ from .compat import (
 from .utils import (
     age_restricted,
     args_to_str,
+    bug_reports_message,
     ContentTooShortError,
     date_from_str,
     DateRange,
@@ -97,6 +101,7 @@ from .utils import (
     std_headers,
     str_or_none,
     subtitles_filename,
+    traverse_obj,
     UnavailableVideoError,
     url_basename,
     version_tuple,
@@ -376,6 +381,9 @@ class YoutubeDL(object):
         self.params.update(params)
         self.cache = Cache(self)
 
+        self._header_cookies = []
+        self._load_cookies_from_headers(self.params.get('http_headers'))
+
         def check_deprecated(param, option, suggestion):
             if self.params.get(param) is not None:
                 self.report_warning(
@@ -870,8 +878,83 @@ class YoutubeDL(object):
                     raise
         return wrapper
 
+    def _remove_cookie_header(self, http_headers):
+        """Filters out `Cookie` header from an `http_headers` dict
+        The `Cookie` header is removed to prevent leaks as a result of unscoped cookies.
+        See: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj
+
+        @param http_headers     An `http_headers` dict from which any `Cookie` header
+                                should be removed, or None
+        """
+        return dict(filter(lambda pair: pair[0].lower() != 'cookie', (http_headers or {}).items()))
+
+    def _load_cookies(self, data, **kwargs):
+        """Loads cookies from a `Cookie` header
+
+        This tries to work around the security vulnerability of passing cookies to every domain.
+
+        @param data         The Cookie header as a string to load the cookies from
+        @param autoscope    If `False`, scope cookies using Set-Cookie syntax and error for cookie without domains
+                            If `True`, save cookies for later to be stored in the jar with a limited scope
+                            If a URL, save cookies in the jar with the domain of the URL
+        """
+        # autoscope=True (kw-only)
+        autoscope = kwargs.get('autoscope', True)
+
+        for cookie in compat_http_cookies_SimpleCookie(data).values() if data else []:
+            if autoscope and any(cookie.values()):
+                raise ValueError('Invalid syntax in Cookie Header')
+
+            domain = cookie.get('domain') or ''
+            expiry = cookie.get('expires')
+            if expiry == '':  # 0 is valid so we check for `''` explicitly
+                expiry = None
+            prepared_cookie = compat_http_cookiejar_Cookie(
+                cookie.get('version') or 0, cookie.key, cookie.value, None, False,
+                domain, True, True, cookie.get('path') or '', bool(cookie.get('path')),
+                bool(cookie.get('secure')), expiry, False, None, None, {})
+
+            if domain:
+                self.cookiejar.set_cookie(prepared_cookie)
+            elif autoscope is True:
+                self.report_warning(
+                    'Passing cookies as a header is a potential security risk; '
+                    'they will be scoped to the domain of the downloaded urls. '
+                    'Please consider loading cookies from a file or browser instead.',
+                    only_once=True)
+                self._header_cookies.append(prepared_cookie)
+            elif autoscope:
+                self.report_warning(
+                    'The extractor result contains an unscoped cookie as an HTTP header. '
+                    'If you are specifying an input URL, ' + bug_reports_message(),
+                    only_once=True)
+                self._apply_header_cookies(autoscope, [prepared_cookie])
+            else:
+                self.report_unscoped_cookies()
+
+    def _load_cookies_from_headers(self, headers):
+        self._load_cookies(traverse_obj(headers, 'cookie', casesense=False))
+
+    def _apply_header_cookies(self, url, cookies=None):
+        """This method applies stray header cookies to the provided url
+
+        This loads header cookies and scopes them to the domain provided in `url`.
+        While this is not ideal, it helps reduce the risk of them being sent to
+        an unintended destination.
+        """
+        parsed = compat_urllib_parse.urlparse(url)
+        if not parsed.hostname:
+            return
+
+        for cookie in map(copy.copy, cookies or self._header_cookies):
+            cookie.domain = '.' + parsed.hostname
+            self.cookiejar.set_cookie(cookie)
+
     @__handle_extraction_exceptions
     def __extract_info(self, url, ie, download, extra_info, process):
+        # Compat with passing cookies in http headers
+        self._apply_header_cookies(url)
+
         ie_result = ie.extract(url)
         if ie_result is None:  # Finished already (backwards compatibility; listformats and friends should be moved here)
             return
@@ -897,7 +980,7 @@ class YoutubeDL(object):
 
     def process_ie_result(self, ie_result, download=True, extra_info={}):
         """
-        Take the result of the ie(may be modified) and resolve all unresolved
+        Take the result of the ie (may be modified) and resolve all unresolved
         references (URLs, playlist items).
 
         It will also download the videos if 'download'.
@@ -1468,23 +1551,45 @@ class YoutubeDL(object):
         parsed_selector = _parse_format_selection(iter(TokenIterator(tokens)))
         return _build_selector_function(parsed_selector)
 
-    def _calc_headers(self, info_dict):
-        res = std_headers.copy()
+    def _calc_headers(self, info_dict, load_cookies=False):
+        if load_cookies:  # For --load-info-json
+            # load cookies from http_headers in legacy info.json
+            self._load_cookies(traverse_obj(info_dict, ('http_headers', 'Cookie'), casesense=False),
+                               autoscope=info_dict['url'])
+            # load scoped cookies from info.json
+            self._load_cookies(info_dict.get('cookies'), autoscope=False)
 
-        add_headers = info_dict.get('http_headers')
-        if add_headers:
-            res.update(add_headers)
-
-        cookies = self._calc_cookies(info_dict)
+        cookies = self.cookiejar.get_cookies_for_url(info_dict['url'])
         if cookies:
-            res['Cookie'] = cookies
+            # Make a string like name1=val1; attr1=a_val1; ...name2=val2; ...
+            # By convention a cookie name can't be a well-known attribute name
+            # so this syntax is unambiguous and can be parsed by (eg) SimpleCookie
+            encoder = compat_http_cookies_SimpleCookie()
+            values = []
+            attributes = (('Domain', '='), ('Path', '='), ('Secure',), ('Expires', '='), ('Version', '='))
+            attributes = tuple([x[0].lower()] + list(x) for x in attributes)
+            for cookie in cookies:
+                _, value = encoder.value_encode(cookie.value)
+                # Py 2 '' --> '', Py 3 '' --> '""'
+                if value == '':
+                    value = '""'
+                values.append('='.join((cookie.name, value)))
+                for attr in attributes:
+                    value = getattr(cookie, attr[0], None)
+                    if value:
+                        values.append('%s%s' % (''.join(attr[1:]), value if len(attr) == 3 else ''))
+            info_dict['cookies'] = '; '.join(values)
+
+        res = std_headers.copy()
+        res.update(info_dict.get('http_headers') or {})
+        res = self._remove_cookie_header(res)
 
         if 'X-Forwarded-For' not in res:
             x_forwarded_for_ip = info_dict.get('__x_forwarded_for_ip')
             if x_forwarded_for_ip:
                 res['X-Forwarded-For'] = x_forwarded_for_ip
 
-        return res
+        return res or None
 
     def _calc_cookies(self, info_dict):
         pr = sanitized_Request(info_dict['url'])
@@ -1663,10 +1768,13 @@ class YoutubeDL(object):
                 format['protocol'] = determine_protocol(format)
             # Add HTTP headers, so that external programs can use them from the
             # json output
-            full_format_info = info_dict.copy()
-            full_format_info.update(format)
-            format['http_headers'] = self._calc_headers(full_format_info)
-        # Remove private housekeeping stuff
+            format['http_headers'] = self._calc_headers(ChainMap(format, info_dict), load_cookies=True)
+
+        # Safeguard against old/insecure infojson when using --load-info-json
+        info_dict['http_headers'] = self._remove_cookie_header(
+            info_dict.get('http_headers') or {}) or None
+
+        # Remove private housekeeping stuff (copied to http_headers in _calc_headers())
         if '__x_forwarded_for_ip' in info_dict:
             del info_dict['__x_forwarded_for_ip']
 
@@ -1927,17 +2035,9 @@ class YoutubeDL(object):
                                                 (sub_lang, error_to_compat_str(err)))
                             continue
 
-        if self.params.get('writeinfojson', False):
-            infofn = replace_extension(filename, 'info.json', info_dict.get('ext'))
-            if self.params.get('nooverwrites', False) and os.path.exists(encodeFilename(infofn)):
-                self.to_screen('[info] Video description metadata is already present')
-            else:
-                self.to_screen('[info] Writing video description metadata as JSON to: ' + infofn)
-                try:
-                    write_json_file(self.filter_requested_info(info_dict), infofn)
-                except (OSError, IOError):
-                    self.report_error('Cannot write metadata to JSON file ' + infofn)
-                    return
+        self._write_info_json(
+            'video description', info_dict,
+            replace_extension(filename, 'info.json', info_dict.get('ext')))
 
         self._write_thumbnails(info_dict, filename)
 
@@ -1958,7 +2058,11 @@ class YoutubeDL(object):
                         fd.add_progress_hook(ph)
                     if self.params.get('verbose'):
                         self.to_screen('[debug] Invoking downloader on %r' % info.get('url'))
-                    return fd.download(name, info)
+
+                    new_info = dict((k, v) for k, v in info.items() if not k.startswith('__p'))
+                    new_info['http_headers'] = self._calc_headers(new_info)
+
+                    return fd.download(name, new_info)
 
                 if info_dict.get('requested_formats') is not None:
                     downloaded = []
@@ -2484,7 +2588,7 @@ class YoutubeDL(object):
         opts_proxy = self.params.get('proxy')
 
         if opts_cookiefile is None:
-            self.cookiejar = compat_cookiejar.CookieJar()
+            self.cookiejar = YoutubeDLCookieJar()
         else:
             opts_cookiefile = expand_path(opts_cookiefile)
             self.cookiejar = YoutubeDLCookieJar(opts_cookiefile)
@@ -2545,6 +2649,28 @@ class YoutubeDL(object):
             encoding = preferredencoding()
         return encoding
 
+    def _write_info_json(self, label, info_dict, infofn, overwrite=None):
+        if not self.params.get('writeinfojson', False):
+            return False
+
+        def msg(fmt, lbl):
+            return fmt % (lbl + ' metadata',)
+
+        if overwrite is None:
+            overwrite = not self.params.get('nooverwrites', False)
+
+        if not overwrite and os.path.exists(encodeFilename(infofn)):
+            self.to_screen(msg('[info] %s is already present', label.title()))
+            return 'exists'
+        else:
+            self.to_screen(msg('[info] Writing %s as JSON to: ' + infofn, label))
+            try:
+                write_json_file(self.filter_requested_info(info_dict), infofn)
+                return True
+            except (OSError, IOError):
+                self.report_error(msg('Cannot write %s to JSON file ' + infofn, label))
+                return
+
     def _write_thumbnails(self, info_dict, filename):
         if self.params.get('writethumbnail', False):
             thumbnails = info_dict.get('thumbnails')
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index c86ce2aa5..08c98b336 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -13,7 +13,9 @@ from ..utils import (
     error_to_compat_str,
     format_bytes,
     shell_quote,
+    T,
     timeconvert,
+    traverse_obj,
 )
 
 
@@ -339,6 +341,10 @@ class FileDownloader(object):
     def download(self, filename, info_dict):
         """Download to a filename using the info from info_dict
         Return True on success and False otherwise
+
+        This method filters the `Cookie` header from the info_dict to prevent leaks.
+        Downloaders have their own way of handling cookies.
+        See: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-v8mc-9377-rwjj
         """
 
         nooverwrites_and_exists = (
@@ -373,6 +379,9 @@ class FileDownloader(object):
                     else '%.2f' % sleep_interval))
             time.sleep(sleep_interval)
 
+        info_dict['http_headers'] = dict(traverse_obj(info_dict, (
+            'http_headers', T(dict.items), lambda _, pair: pair[0].lower() != 'cookie'))) or None
+
         return self.real_download(filename, info_dict)
 
     def real_download(self, filename, info_dict):

From 21438a4194376c3a9b1e5c322c825d43a1b03d6e Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@bashonly.com>
Date: Tue, 4 Jul 2023 16:40:56 -0500
Subject: [PATCH 669/860] [downloader/external] Fix cookie support

---
 test/test_downloader_external.py  | 157 ++++++++++++++++++++++++++++--
 youtube_dl/downloader/common.py   |   5 -
 youtube_dl/downloader/external.py | 124 +++++++++++++++++++----
 3 files changed, 256 insertions(+), 30 deletions(-)

diff --git a/test/test_downloader_external.py b/test/test_downloader_external.py
index c0239502b..029f9b05f 100644
--- a/test/test_downloader_external.py
+++ b/test/test_downloader_external.py
@@ -12,20 +12,65 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from test.helper import (
     FakeLogger,
+    FakeYDL,
     http_server_port,
     try_rm,
 )
 from youtube_dl import YoutubeDL
-from youtube_dl.compat import compat_http_server
-from youtube_dl.utils import encodeFilename
-from youtube_dl.downloader.external import Aria2pFD
+from youtube_dl.compat import (
+    compat_http_cookiejar_Cookie,
+    compat_http_server,
+    compat_kwargs,
+)
+from youtube_dl.utils import (
+    encodeFilename,
+    join_nonempty,
+)
+from youtube_dl.downloader.external import (
+    Aria2cFD,
+    Aria2pFD,
+    AxelFD,
+    CurlFD,
+    FFmpegFD,
+    HttpieFD,
+    WgetFD,
+)
 import threading
 
-TEST_DIR = os.path.dirname(os.path.abspath(__file__))
-
-
 TEST_SIZE = 10 * 1024
 
+TEST_COOKIE = {
+    'version': 0,
+    'name': 'test',
+    'value': 'ytdlp',
+    'port': None,
+    'port_specified': False,
+    'domain': '.example.com',
+    'domain_specified': True,
+    'domain_initial_dot': False,
+    'path': '/',
+    'path_specified': True,
+    'secure': False,
+    'expires': None,
+    'discard': False,
+    'comment': None,
+    'comment_url': None,
+    'rest': {},
+}
+
+TEST_COOKIE_VALUE = join_nonempty('name', 'value', delim='=', from_dict=TEST_COOKIE)
+
+TEST_INFO = {'url': 'http://www.example.com/'}
+
+
+def cookiejar_Cookie(**cookie_args):
+    return compat_http_cookiejar_Cookie(**compat_kwargs(cookie_args))
+
+
+def ifExternalFDAvailable(externalFD):
+    return unittest.skipUnless(externalFD.available(),
+                               externalFD.get_basename() + ' not found')
+
 
 class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
     def log_message(self, format, *args):
@@ -70,7 +115,7 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
             assert False, 'unrecognised server path'
 
 
-@unittest.skipUnless(Aria2pFD.available(), 'aria2p module not found')
+@ifExternalFDAvailable(Aria2pFD)
 class TestAria2pFD(unittest.TestCase):
     def setUp(self):
         self.httpd = compat_http_server.HTTPServer(
@@ -111,5 +156,103 @@ class TestAria2pFD(unittest.TestCase):
         })
 
 
+@ifExternalFDAvailable(HttpieFD)
+class TestHttpieFD(unittest.TestCase):
+    def test_make_cmd(self):
+        with FakeYDL() as ydl:
+            downloader = HttpieFD(ydl, {})
+            self.assertEqual(
+                downloader._make_cmd('test', TEST_INFO),
+                ['http', '--download', '--output', 'test', 'http://www.example.com/'])
+
+            # Test cookie header is added
+            ydl.cookiejar.set_cookie(cookiejar_Cookie(**TEST_COOKIE))
+            self.assertEqual(
+                downloader._make_cmd('test', TEST_INFO),
+                ['http', '--download', '--output', 'test',
+                 'http://www.example.com/', 'Cookie:' + TEST_COOKIE_VALUE])
+
+
+@ifExternalFDAvailable(AxelFD)
+class TestAxelFD(unittest.TestCase):
+    def test_make_cmd(self):
+        with FakeYDL() as ydl:
+            downloader = AxelFD(ydl, {})
+            self.assertEqual(
+                downloader._make_cmd('test', TEST_INFO),
+                ['axel', '-o', 'test', '--', 'http://www.example.com/'])
+
+            # Test cookie header is added
+            ydl.cookiejar.set_cookie(cookiejar_Cookie(**TEST_COOKIE))
+            self.assertEqual(
+                downloader._make_cmd('test', TEST_INFO),
+                ['axel', '-o', 'test', '-H', 'Cookie: ' + TEST_COOKIE_VALUE,
+                 '--max-redirect=0', '--', 'http://www.example.com/'])
+
+
+@ifExternalFDAvailable(WgetFD)
+class TestWgetFD(unittest.TestCase):
+    def test_make_cmd(self):
+        with FakeYDL() as ydl:
+            downloader = WgetFD(ydl, {})
+            self.assertNotIn('--load-cookies', downloader._make_cmd('test', TEST_INFO))
+            # Test cookiejar tempfile arg is added
+            ydl.cookiejar.set_cookie(cookiejar_Cookie(**TEST_COOKIE))
+            self.assertIn('--load-cookies', downloader._make_cmd('test', TEST_INFO))
+
+
+@ifExternalFDAvailable(CurlFD)
+class TestCurlFD(unittest.TestCase):
+    def test_make_cmd(self):
+        with FakeYDL() as ydl:
+            downloader = CurlFD(ydl, {})
+            self.assertNotIn('--cookie', downloader._make_cmd('test', TEST_INFO))
+            # Test cookie header is added
+            ydl.cookiejar.set_cookie(cookiejar_Cookie(**TEST_COOKIE))
+            self.assertIn('--cookie', downloader._make_cmd('test', TEST_INFO))
+            self.assertIn(TEST_COOKIE_VALUE, downloader._make_cmd('test', TEST_INFO))
+
+
+@ifExternalFDAvailable(Aria2cFD)
+class TestAria2cFD(unittest.TestCase):
+    def test_make_cmd(self):
+        with FakeYDL() as ydl:
+            downloader = Aria2cFD(ydl, {})
+            downloader._make_cmd('test', TEST_INFO)
+            self.assertFalse(hasattr(downloader, '_cookies_tempfile'))
+
+            # Test cookiejar tempfile arg is added
+            ydl.cookiejar.set_cookie(cookiejar_Cookie(**TEST_COOKIE))
+            cmd = downloader._make_cmd('test', TEST_INFO)
+            self.assertIn('--load-cookies=%s' % downloader._cookies_tempfile, cmd)
+
+
+@ifExternalFDAvailable(FFmpegFD)
+class TestFFmpegFD(unittest.TestCase):
+    _args = []
+
+    def _test_cmd(self, args):
+        self._args = args
+
+    def test_make_cmd(self):
+        with FakeYDL() as ydl:
+            downloader = FFmpegFD(ydl, {})
+            downloader._debug_cmd = self._test_cmd
+            info_dict = TEST_INFO.copy()
+            info_dict['ext'] = 'mp4'
+
+            downloader._call_downloader('test', info_dict)
+            self.assertEqual(self._args, [
+                'ffmpeg', '-y', '-i', 'http://www.example.com/',
+                '-c', 'copy', '-f', 'mp4', 'file:test'])
+
+            # Test cookies arg is added
+            ydl.cookiejar.set_cookie(cookiejar_Cookie(**TEST_COOKIE))
+            downloader._call_downloader('test', info_dict)
+            self.assertEqual(self._args, [
+                'ffmpeg', '-y', '-cookies', TEST_COOKIE_VALUE + '; path=/; domain=.example.com;\r\n',
+                '-i', 'http://www.example.com/', '-c', 'copy', '-f', 'mp4', 'file:test'])
+
+
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index 08c98b336..afb4ee33d 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -13,9 +13,7 @@ from ..utils import (
     error_to_compat_str,
     format_bytes,
     shell_quote,
-    T,
     timeconvert,
-    traverse_obj,
 )
 
 
@@ -379,9 +377,6 @@ class FileDownloader(object):
                     else '%.2f' % sleep_interval))
             time.sleep(sleep_interval)
 
-        info_dict['http_headers'] = dict(traverse_obj(info_dict, (
-            'http_headers', T(dict.items), lambda _, pair: pair[0].lower() != 'cookie'))) or None
-
         return self.real_download(filename, info_dict)
 
     def real_download(self, filename, info_dict):
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 1b6bd1fa2..7fc864e85 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -1,9 +1,10 @@
 from __future__ import unicode_literals
 
-import os.path
+import os
 import re
 import subprocess
 import sys
+import tempfile
 import time
 
 from .common import FileDownloader
@@ -23,6 +24,8 @@ from ..utils import (
     check_executable,
     is_outdated_version,
     process_communicate_or_kill,
+    T,
+    traverse_obj,
 )
 
 
@@ -30,6 +33,7 @@ class ExternalFD(FileDownloader):
     def real_download(self, filename, info_dict):
         self.report_destination(filename)
         tmpfilename = self.temp_name(filename)
+        self._cookies_tempfile = None
 
         try:
             started = time.time()
@@ -42,6 +46,13 @@ class ExternalFD(FileDownloader):
             # should take place
             retval = 0
             self.to_screen('[%s] Interrupted by user' % self.get_basename())
+        finally:
+            if self._cookies_tempfile and os.path.isfile(self._cookies_tempfile):
+                try:
+                    os.remove(self._cookies_tempfile)
+                except OSError:
+                    self.report_warning(
+                        'Unable to delete temporary cookies file "{0}"'.format(self._cookies_tempfile))
 
         if retval == 0:
             status = {
@@ -97,6 +108,16 @@ class ExternalFD(FileDownloader):
     def _configuration_args(self, default=[]):
         return cli_configuration_args(self.params, 'external_downloader_args', default)
 
+    def _write_cookies(self):
+        if not self.ydl.cookiejar.filename:
+            tmp_cookies = tempfile.NamedTemporaryFile(suffix='.cookies', delete=False)
+            tmp_cookies.close()
+            self._cookies_tempfile = tmp_cookies.name
+            self.to_screen('[download] Writing temporary cookies file to "{0}"'.format(self._cookies_tempfile))
+        # real_download resets _cookies_tempfile; if it's None, save() will write to cookiejar.filename
+        self.ydl.cookiejar.save(self._cookies_tempfile, ignore_discard=True, ignore_expires=True)
+        return self.ydl.cookiejar.filename or self._cookies_tempfile
+
     def _call_downloader(self, tmpfilename, info_dict):
         """ Either overwrite this or implement _make_cmd """
         cmd = [encodeArgument(a) for a in self._make_cmd(tmpfilename, info_dict)]
@@ -110,13 +131,21 @@ class ExternalFD(FileDownloader):
             self.to_stderr(stderr.decode('utf-8', 'replace'))
         return p.returncode
 
+    @staticmethod
+    def _header_items(info_dict):
+        return traverse_obj(
+            info_dict, ('http_headers', T(dict.items), Ellipsis))
+
 
 class CurlFD(ExternalFD):
     AVAILABLE_OPT = '-V'
 
     def _make_cmd(self, tmpfilename, info_dict):
-        cmd = [self.exe, '--location', '-o', tmpfilename]
-        for key, val in info_dict['http_headers'].items():
+        cmd = [self.exe, '--location', '-o', tmpfilename, '--compressed']
+        cookie_header = self.ydl.cookiejar.get_cookie_header(info_dict['url'])
+        if cookie_header:
+            cmd += ['--cookie', cookie_header]
+        for key, val in self._header_items(info_dict):
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._bool_option('--continue-at', 'continuedl', '-', '0')
         cmd += self._valueless_option('--silent', 'noprogress')
@@ -151,8 +180,11 @@ class AxelFD(ExternalFD):
 
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = [self.exe, '-o', tmpfilename]
-        for key, val in info_dict['http_headers'].items():
+        for key, val in self._header_items(info_dict):
             cmd += ['-H', '%s: %s' % (key, val)]
+        cookie_header = self.ydl.cookiejar.get_cookie_header(info_dict['url'])
+        if cookie_header:
+            cmd += ['-H', 'Cookie: {0}'.format(cookie_header), '--max-redirect=0']
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]
         return cmd
@@ -162,8 +194,10 @@ class WgetFD(ExternalFD):
     AVAILABLE_OPT = '--version'
 
     def _make_cmd(self, tmpfilename, info_dict):
-        cmd = [self.exe, '-O', tmpfilename, '-nv', '--no-cookies']
-        for key, val in info_dict['http_headers'].items():
+        cmd = [self.exe, '-O', tmpfilename, '-nv', '--compression=auto']
+        if self.ydl.cookiejar.get_cookie_header(info_dict['url']):
+            cmd += ['--load-cookies', self._write_cookies()]
+        for key, val in self._header_items(info_dict):
             cmd += ['--header', '%s: %s' % (key, val)]
         cmd += self._option('--limit-rate', 'ratelimit')
         retry = self._option('--tries', 'retries')
@@ -182,21 +216,58 @@ class WgetFD(ExternalFD):
 class Aria2cFD(ExternalFD):
     AVAILABLE_OPT = '-v'
 
+    @staticmethod
+    def _aria2c_filename(fn):
+        return fn if os.path.isabs(fn) else os.path.join('.', fn)
+
     def _make_cmd(self, tmpfilename, info_dict):
-        cmd = [self.exe, '-c']
-        cmd += self._configuration_args([
-            '--min-split-size', '1M', '--max-connection-per-server', '4'])
-        dn = os.path.dirname(tmpfilename)
-        if dn:
-            cmd += ['--dir', dn]
-        cmd += ['--out', os.path.basename(tmpfilename)]
-        for key, val in info_dict['http_headers'].items():
+        cmd = [self.exe, '-c',
+               '--console-log-level=warn', '--summary-interval=0', '--download-result=hide',
+               '--http-accept-gzip=true', '--file-allocation=none', '-x16', '-j16', '-s16']
+        if 'fragments' in info_dict:
+            cmd += ['--allow-overwrite=true', '--allow-piece-length-change=true']
+        else:
+            cmd += ['--min-split-size', '1M']
+
+        if self.ydl.cookiejar.get_cookie_header(info_dict['url']):
+            cmd += ['--load-cookies={0}'.format(self._write_cookies())]
+        for key, val in self._header_items(info_dict):
             cmd += ['--header', '%s: %s' % (key, val)]
+        cmd += self._configuration_args(['--max-connection-per-server', '4'])
+        cmd += ['--out', os.path.basename(tmpfilename)]
+        cmd += self._option('--max-overall-download-limit', 'ratelimit')
         cmd += self._option('--interface', 'source_address')
         cmd += self._option('--all-proxy', 'proxy')
         cmd += self._bool_option('--check-certificate', 'nocheckcertificate', 'false', 'true', '=')
         cmd += self._bool_option('--remote-time', 'updatetime', 'true', 'false', '=')
-        cmd += ['--', info_dict['url']]
+        cmd += self._bool_option('--show-console-readout', 'noprogress', 'false', 'true', '=')
+        cmd += self._configuration_args()
+
+        # aria2c strips out spaces from the beginning/end of filenames and paths.
+        # We work around this issue by adding a "./" to the beginning of the
+        # filename and relative path, and adding a "/" at the end of the path.
+        # See: https://github.com/yt-dlp/yt-dlp/issues/276
+        # https://github.com/ytdl-org/youtube-dl/issues/20312
+        # https://github.com/aria2/aria2/issues/1373
+        dn = os.path.dirname(tmpfilename)
+        if dn:
+            cmd += ['--dir', self._aria2c_filename(dn) + os.path.sep]
+        if 'fragments' not in info_dict:
+            cmd += ['--out', self._aria2c_filename(os.path.basename(tmpfilename))]
+        cmd += ['--auto-file-renaming=false']
+        if 'fragments' in info_dict:
+            cmd += ['--file-allocation=none', '--uri-selector=inorder']
+            url_list_file = '%s.frag.urls' % (tmpfilename, )
+            url_list = []
+            for frag_index, fragment in enumerate(info_dict['fragments']):
+                fragment_filename = '%s-Frag%d' % (os.path.basename(tmpfilename), frag_index)
+                url_list.append('%s\n\tout=%s' % (fragment['url'], self._aria2c_filename(fragment_filename)))
+            stream, _ = self.sanitize_open(url_list_file, 'wb')
+            stream.write('\n'.join(url_list).encode())
+            stream.close()
+            cmd += ['-i', self._aria2c_filename(url_list_file)]
+        else:
+            cmd += ['--', info_dict['url']]
         return cmd
 
 
@@ -235,8 +306,10 @@ class Aria2pFD(ExternalFD):
         }
         options['dir'] = os.path.dirname(tmpfilename) or os.path.abspath('.')
         options['out'] = os.path.basename(tmpfilename)
+        if self.ydl.cookiejar.get_cookie_header(info_dict['url']):
+            options['load-cookies'] = self._write_cookies()
         options['header'] = []
-        for key, val in info_dict['http_headers'].items():
+        for key, val in self._header_items(info_dict):
             options['header'].append('{0}: {1}'.format(key, val))
         download = aria2.add_uris([info_dict['url']], options)
         status = {
@@ -265,8 +338,16 @@ class HttpieFD(ExternalFD):
 
     def _make_cmd(self, tmpfilename, info_dict):
         cmd = ['http', '--download', '--output', tmpfilename, info_dict['url']]
-        for key, val in info_dict['http_headers'].items():
+        for key, val in self._header_items(info_dict):
             cmd += ['%s:%s' % (key, val)]
+
+        # httpie 3.1.0+ removes the Cookie header on redirect, so this should be safe for now. [1]
+        # If we ever need cookie handling for redirects, we can export the cookiejar into a session. [2]
+        # 1: https://github.com/httpie/httpie/security/advisories/GHSA-9w4w-cpc8-h2fq
+        # 2: https://httpie.io/docs/cli/sessions
+        cookie_header = self.ydl.cookiejar.get_cookie_header(info_dict['url'])
+        if cookie_header:
+            cmd += ['Cookie:%s' % cookie_header]
         return cmd
 
 
@@ -312,7 +393,14 @@ class FFmpegFD(ExternalFD):
         # if end_time:
         #     args += ['-t', compat_str(end_time - start_time)]
 
-        if info_dict['http_headers'] and re.match(r'^https?://', url):
+        cookies = self.ydl.cookiejar.get_cookies_for_url(url)
+        if cookies:
+            args.extend(['-cookies', ''.join(
+                '{0}={1}; path={2}; domain={3};\r\n'.format(
+                    cookie.name, cookie.value, cookie.path, cookie.domain)
+                for cookie in cookies)])
+
+        if info_dict.get('http_headers') and re.match(r'^https?://', url):
             # Trailing \r\n after each HTTP header is important to prevent warning from ffmpeg/avconv:
             # [http @ 00000000003d2fa0] No trailing CRLF found in HTTP header.
             headers = handle_youtubedl_headers(info_dict['http_headers'])

From 1634b1d61efa36c31c86b8c64c88dc297a7af28a Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 11 Jul 2023 21:51:32 +0100
Subject: [PATCH 670/860] [doc] Warn against setting cookies with --add-header

---
 youtube_dl/options.py | 6 ++++--
 1 file changed, 4 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index d802b7e59..434f520d3 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -544,12 +544,14 @@ def parseOpts(overrideArguments=None):
     workarounds.add_option(
         '--referer',
         metavar='URL', dest='referer', default=None,
-        help='Specify a custom referer, use if the video access is restricted to one domain',
+        help='Specify a custom Referer: use if the video access is restricted to one domain',
     )
     workarounds.add_option(
         '--add-header',
         metavar='FIELD:VALUE', dest='headers', action='append',
-        help='Specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times',
+        help=('Specify a custom HTTP header and its value, separated by a colon \':\'. You can use this option multiple times. '
+              'NB Use --cookies rather than adding a Cookie header if its contents may be sensitive; '
+              'data from a Cookie header will be sent to all domains, not just the one intended')
     )
     workarounds.add_option(
         '--bidi-workaround',

From 1d8d5a93f7187438587c3a754b53fdf30322cef0 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 13 Jul 2023 20:14:50 +0100
Subject: [PATCH 671/860] [test] Fixes for old Pythons

---
 .github/workflows/ci.yml |  4 ++--
 test/helper.py           |  6 ++++++
 test/test_http.py        |  7 ++++++-
 test/test_utils.py       |  6 +++---
 youtube_dl/jsinterp.py   | 14 +++++++-------
 youtube_dl/utils.py      |  8 ++++++--
 6 files changed, 30 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ce878c1b1..c3aabde47 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -301,7 +301,7 @@ jobs:
       if: ${{ matrix.python-version == '2.6' }}
       shell: bash
       run: |
-        # see pip for Jython
+        # Work around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb)
         $PIP -qq show unittest2 || { \
           for u in "65/26/32b8464df2a97e6dd1b656ed26b2c194606c16fe163c695a992b36c11cdf/six-1.13.0-py2.py3-none-any.whl" \
               "f2/94/3af39d34be01a24a6e65433d19e107099374224905f1e0cc6bbe1fd22a2f/argparse-1.4.0-py2.py3-none-any.whl" \
@@ -312,7 +312,7 @@ jobs:
             $PIP install ${u##*/}; \
           done; }
         # make tests use unittest2
-        for test in ./test/test_*.py; do
+        for test in ./test/test_*.py ./test/helper.py; do
           sed -r -i -e '/^import unittest$/s/test/test2 as unittest/' "$test"
         done
     #-------- nose --------
diff --git a/test/helper.py b/test/helper.py
index e3314b03e..aa99001b2 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -9,6 +9,7 @@ import re
 import types
 import ssl
 import sys
+import unittest
 
 import youtube_dl.extractor
 from youtube_dl import YoutubeDL
@@ -17,6 +18,7 @@ from youtube_dl.compat import (
     compat_str,
 )
 from youtube_dl.utils import (
+    IDENTITY,
     preferredencoding,
     write_string,
 )
@@ -298,3 +300,7 @@ def http_server_port(httpd):
     else:
         sock = httpd.socket
     return sock.getsockname()[1]
+
+
+def expectedFailureIf(cond):
+    return unittest.expectedFailure if cond else IDENTITY
diff --git a/test/test_http.py b/test/test_http.py
index cd180b51f..1a6b2e878 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -8,6 +8,7 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+import contextlib
 import gzip
 import io
 import ssl
@@ -154,7 +155,7 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
 
         def gzip_compress(p):
             buf = io.BytesIO()
-            with gzip.GzipFile(fileobj=buf, mode='wb') as f:
+            with contextlib.closing(gzip.GzipFile(fileobj=buf, mode='wb')) as f:
                 f.write(p)
             return buf.getvalue()
 
@@ -306,6 +307,10 @@ class TestHTTP(unittest.TestCase):
             else self.https_port if scheme == 'https'
             else self.http_port, path)
 
+    @unittest.skipUnless(
+        sys.version_info >= (3, 2)
+        or (sys.version_info[0] == 2 and sys.version_info[1:] >= (7, 9)),
+        'No support for certificate check in SSL')
     def test_nocheckcertificate(self):
         with FakeYDL({'logger': FakeLogger()}) as ydl:
             with self.assertRaises(compat_urllib_error.URLError):
diff --git a/test/test_utils.py b/test/test_utils.py
index 1fc16ed05..2ee727caf 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -1617,7 +1617,7 @@ Line 1
         self.assertEqual(traverse_obj(_TEST_DATA, lambda x, y: x == 'urls' and isinstance(y, list)),
                          [_TEST_DATA['urls']],
                          msg='function as query key should perform a filter based on (key, value)')
-        self.assertCountEqual(traverse_obj(_TEST_DATA, lambda _, x: isinstance(x[0], str)), {'str'},
+        self.assertCountEqual(traverse_obj(_TEST_DATA, lambda _, x: isinstance(x[0], str)), set(('str',)),
                               msg='exceptions in the query function should be catched')
         self.assertEqual(traverse_obj(iter(range(4)), lambda _, x: x % 2 == 0), [0, 2],
                          msg='function key should accept iterables')
@@ -1643,7 +1643,7 @@ Line 1
             with self.assertRaises(Exception, msg='Sets with length != 1 should raise in debug'):
                 traverse_obj(_TEST_DATA, set())
             with self.assertRaises(Exception, msg='Sets with length != 1 should raise in debug'):
-                traverse_obj(_TEST_DATA, {str.upper, str})
+                traverse_obj(_TEST_DATA, set((str.upper, str)))
 
         # Test `slice` as a key
         _SLICE_DATA = [0, 1, 2, 3, 4]
@@ -1779,7 +1779,7 @@ Line 1
                          {0: 100}, msg='type as expected_type should filter dict values')
         self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2, 2: 'None'}, expected_type=str_or_none),
                          {0: '100', 1: '1.2'}, msg='function as expected_type should transform dict values')
-        self.assertEqual(traverse_obj(_TEST_DATA, ({0: 1.2}, 0, {int_or_none}), expected_type=int),
+        self.assertEqual(traverse_obj(_TEST_DATA, ({0: 1.2}, 0, set((int_or_none,))), expected_type=int),
                          1, msg='expected_type should not filter non final dict values')
         self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 100, 1: 'str'}}, expected_type=int),
                          {0: {0: 100}}, msg='expected_type should transform deep dict values')
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 882432b80..86d902248 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -280,16 +280,16 @@ class JSInterpreter(object):
             # make Py 2.6 conform to its lying documentation
             if name == 'flags':
                 self.flags = self.__flags
+                return self.flags
             elif name == 'pattern':
                 self.pattern = self.__pattern_txt
+                return self.pattern
+            elif hasattr(self.__self, name):
+                v = getattr(self.__self, name)
+                setattr(self, name, v)
+                return v
             elif name in ('groupindex', 'groups'):
-                # in case these get set after a match?
-                if hasattr(self.__self, name):
-                    setattr(self, name, getattr(self.__self, name))
-                else:
-                    return 0 if name == 'groupindex' else {}
-            if hasattr(self, name):
-                return getattr(self, name)
+                return 0 if name == 'groupindex' else {}
             raise AttributeError('{0} has no attribute named {1}'.format(self, name))
 
         @classmethod
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ac6c81465..494f8341b 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -6198,7 +6198,8 @@ def traverse_obj(obj, *paths, **kwargs):
         elif isinstance(obj, compat_re_Match):
             result = None
             if isinstance(key, int) or casesense:
-                result = lookup_or_none(obj, key, getter=compat_re_Match.group)
+                # Py 2.6 doesn't have methods in the Match class/type
+                result = lookup_or_none(obj, key, getter=lambda _, k: obj.group(k))
 
             elif isinstance(key, str):
                 result = next((v for k, v in obj.groupdict().items()
@@ -6246,7 +6247,10 @@ def traverse_obj(obj, *paths, **kwargs):
 
             if __debug__ and callable(key):
                 # Verify function signature
-                inspect.getcallargs(key, None, None)
+                args = inspect.getargspec(key)
+                if len(args.args) != 2:
+                    # crash differently in 2.6 !
+                    inspect.getcallargs(key, None, None)
 
             new_objs = []
             for obj in objs:

From 47214e46d852e9d7ddf81d69a8e70806e2396c6c Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 17 Jul 2023 20:39:11 +0100
Subject: [PATCH 672/860] [compat] Fix old Pythons broken loading of valueless
 cookie attributes

Cookie string parsing in Py 2.6.9, probably earlier, requires `=`.
Also 3.2, though the CPython code appears to be OK: 3.1 was also wrong.
---
 youtube_dl/compat.py | 20 ++++++++++++++++----
 1 file changed, 16 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index cd11ba5aa..1d784d90f 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -126,12 +126,24 @@ except ImportError:  # Python 2
     import Cookie as compat_cookies
 compat_http_cookies = compat_cookies
 
-if sys.version_info[0] == 2:
+if sys.version_info[0] == 2 or sys.version_info < (3, 3):
     class compat_cookies_SimpleCookie(compat_cookies.SimpleCookie):
         def load(self, rawdata):
-            if isinstance(rawdata, compat_str):
-                rawdata = str(rawdata)
-            return super(compat_cookies_SimpleCookie, self).load(rawdata)
+            must_have_value = 0
+            if not isinstance(rawdata, dict):
+                if sys.version_info[:2] != (2, 7):
+                    # attribute must have value for parsing
+                    rawdata, must_have_value = re.subn(
+                        r'(?i)(;\s*)(secure|httponly)(\s*(?:;|$))', r'\1\2=\2\3', rawdata)
+                if sys.version_info[0] == 2:
+                    if isinstance(rawdata, compat_str):
+                        rawdata = str(rawdata)
+            super(compat_cookies_SimpleCookie, self).load(rawdata)
+            if must_have_value > 0:
+                for morsel in self.values():
+                    for attr in ('secure', 'httponly'):
+                        if morsel.get(attr):
+                            morsel[attr] = True
 else:
     compat_cookies_SimpleCookie = compat_cookies.SimpleCookie
 compat_http_cookies_SimpleCookie = compat_cookies_SimpleCookie

From 825a40744bf9aeb743452db24e43d3eb61feb6c2 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 3 May 2023 12:40:09 +0100
Subject: [PATCH 673/860] [utils] Align traverse_obj() with yt-dlp

Thanks Grub4k for these:
* traverse `Iterable`s, from https://github.com/yt-dlp/yt-dlp/pull/6902, etc
* traverse `set` key for transformations/filters, `re.Match` group names, from
  https://github.com/yt-dlp/yt-dlp/commit/776995bc109c5cd1aa56b684fada2ce718a386ec, etc
* traverse `re.Match`es, from https://github.com/yt-dlp/yt-dlp/pull/5174
* always return list when branching, from https://github.com/yt-dlp/yt-dlp/pull/5170
---
 test/test_utils.py  | 37 +++++++++++++++++++++----------------
 youtube_dl/utils.py |  9 ++-------
 2 files changed, 23 insertions(+), 23 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 2ee727caf..1b5d170fe 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -20,7 +20,7 @@ import xml.etree.ElementTree
 from youtube_dl.utils import (
     age_restricted,
     args_to_str,
-    encode_base_n,
+    base_url,
     caesar,
     clean_html,
     clean_podcast_url,
@@ -29,10 +29,12 @@ from youtube_dl.utils import (
     detect_exe_version,
     determine_ext,
     dict_get,
+    encode_base_n,
     encode_compat_str,
     encodeFilename,
     escape_rfc3986,
     escape_url,
+    expand_path,
     extract_attributes,
     ExtractorError,
     find_xpath_attr,
@@ -51,6 +53,7 @@ from youtube_dl.utils import (
     js_to_json,
     LazyList,
     limit_length,
+    lowercase_escape,
     merge_dicts,
     mimetype2ext,
     month_by_name,
@@ -66,17 +69,16 @@ from youtube_dl.utils import (
     parse_resolution,
     parse_bitrate,
     pkcs1pad,
-    read_batch_urls,
-    sanitize_filename,
-    sanitize_path,
-    sanitize_url,
-    expand_path,
     prepend_extension,
-    replace_extension,
+    read_batch_urls,
     remove_start,
     remove_end,
     remove_quotes,
+    replace_extension,
     rot47,
+    sanitize_filename,
+    sanitize_path,
+    sanitize_url,
     shell_quote,
     smuggle_url,
     str_or_none,
@@ -93,10 +95,8 @@ from youtube_dl.utils import (
     unified_timestamp,
     unsmuggle_url,
     uppercase_escape,
-    lowercase_escape,
     url_basename,
     url_or_none,
-    base_url,
     urljoin,
     urlencode_postdata,
     urshift,
@@ -1586,6 +1586,11 @@ Line 1
             'dict': {},
         }
 
+        # define a pukka Iterable
+        def iter_range(stop):
+            for from_ in range(stop):
+                yield from_
+
         # Test base functionality
         self.assertEqual(traverse_obj(_TEST_DATA, ('str',)), 'str',
                          msg='allow tuple path')
@@ -1602,13 +1607,13 @@ Line 1
         # Test Ellipsis behavior
         self.assertCountEqual(traverse_obj(_TEST_DATA, Ellipsis),
                               (item for item in _TEST_DATA.values() if item not in (None, {})),
-                              msg='`...` should give all non discarded values')
+                              msg='`...` should give all non-discarded values')
         self.assertCountEqual(traverse_obj(_TEST_DATA, ('urls', 0, Ellipsis)), _TEST_DATA['urls'][0].values(),
                               msg='`...` selection for dicts should select all values')
         self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, Ellipsis, 'url')),
                          ['https://www.example.com/0', 'https://www.example.com/1'],
                          msg='nested `...` queries should work')
-        self.assertCountEqual(traverse_obj(_TEST_DATA, (Ellipsis, Ellipsis, 'index')), range(4),
+        self.assertCountEqual(traverse_obj(_TEST_DATA, (Ellipsis, Ellipsis, 'index')), iter_range(4),
                               msg='`...` query result should be flattened')
         self.assertEqual(traverse_obj(iter(range(4)), Ellipsis), list(range(4)),
                          msg='`...` should accept iterables')
@@ -1618,7 +1623,7 @@ Line 1
                          [_TEST_DATA['urls']],
                          msg='function as query key should perform a filter based on (key, value)')
         self.assertCountEqual(traverse_obj(_TEST_DATA, lambda _, x: isinstance(x[0], str)), set(('str',)),
-                              msg='exceptions in the query function should be catched')
+                              msg='exceptions in the query function should be caught')
         self.assertEqual(traverse_obj(iter(range(4)), lambda _, x: x % 2 == 0), [0, 2],
                          msg='function key should accept iterables')
         if __debug__:
@@ -1706,7 +1711,7 @@ Line 1
         self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}), {},
                          msg='remove empty values when dict key')
         self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}, default=Ellipsis), {0: Ellipsis},
-                         msg='use `default` when dict key and `default`')
+                         msg='use `default` when dict key and a default')
         self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 'fail'}}), {},
                          msg='remove empty values when nested dict key fails')
         self.assertEqual(traverse_obj(None, {0: 'fail'}), {},
@@ -1768,7 +1773,7 @@ Line 1
         self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=str),
                          'str', msg='accept matching `expected_type` type')
         self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=int),
-                         None, msg='reject non matching `expected_type` type')
+                         None, msg='reject non-matching `expected_type` type')
         self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'int', expected_type=lambda x: str(x)),
                          '0', msg='transform type using type function')
         self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=lambda _: 1 / 0),
@@ -1780,7 +1785,7 @@ Line 1
         self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2, 2: 'None'}, expected_type=str_or_none),
                          {0: '100', 1: '1.2'}, msg='function as expected_type should transform dict values')
         self.assertEqual(traverse_obj(_TEST_DATA, ({0: 1.2}, 0, set((int_or_none,))), expected_type=int),
-                         1, msg='expected_type should not filter non final dict values')
+                         1, msg='expected_type should not filter non-final dict values')
         self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 100, 1: 'str'}}, expected_type=int),
                          {0: {0: 100}}, msg='expected_type should transform deep dict values')
         self.assertEqual(traverse_obj(_TEST_DATA, [({0: '...'}, {0: '...'})], expected_type=type(Ellipsis)),
@@ -1838,7 +1843,7 @@ Line 1
         self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', slice(0, None, 2)),
                                       _traverse_string=True), 'sr',
                          msg='`slice` should result in string if `traverse_string`')
-        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', lambda i, v: i or v == "s"),
+        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', lambda i, v: i or v == 's'),
                                       _traverse_string=True), 'str',
                          msg='function should result in string if `traverse_string`')
         self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', (0, 2)),
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 494f8341b..b77a7fb0e 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -4268,13 +4268,8 @@ def variadic(x, allowed_types=NO_DEFAULT):
 
 
 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
-    if isinstance(key_or_keys, (list, tuple)):
-        for key in key_or_keys:
-            if key not in d or d[key] is None or skip_false_values and not d[key]:
-                continue
-            return d[key]
-        return default
-    return d.get(key_or_keys, default)
+    exp = (lambda x: x or None) if skip_false_values else IDENTITY
+    return traverse_obj(d, *variadic(key_or_keys), expected_type=exp, default=default)
 
 
 def try_call(*funcs, **kwargs):

From d9d07a95815a992bf5f876a62f25c831eb3f32ac Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 3 May 2023 12:06:34 +0100
Subject: [PATCH 674/860] [utils] Improve js_to_json, align with yt-dlp *
 support variable substitution, from
 https://github.com/yt-dlp/yt-dlp/pull/#521 etc,   thanks ChillingPepper,
 Grub4k, pukkandan * improve escape handling, from
 https://github.com/yt-dlp/yt-dlp/pull/#521   thanks Grub4k * support template
 strings from https://github.com/yt-dlp/yt-dlp/pull/6623   thanks Grub4k * add
 limited `!` evaluation (eg, !!0 -> false, see tests)

---
 test/test_utils.py  | 103 ++++++++++++++++++++++++++++++++++++++--
 youtube_dl/utils.py | 112 ++++++++++++++++++++++++++++++++++----------
 2 files changed, 186 insertions(+), 29 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 1b5d170fe..e83977f29 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -905,6 +905,85 @@ class TestUtil(unittest.TestCase):
         )
         self.assertEqual(escape_url('http://vimeo.com/56015672#at=0'), 'http://vimeo.com/56015672#at=0')
 
+    def test_js_to_json_vars_strings(self):
+        self.assertDictEqual(
+            json.loads(js_to_json(
+                '''{
+                    'null': a,
+                    'nullStr': b,
+                    'true': c,
+                    'trueStr': d,
+                    'false': e,
+                    'falseStr': f,
+                    'unresolvedVar': g,
+                }''',
+                {
+                    'a': 'null',
+                    'b': '"null"',
+                    'c': 'true',
+                    'd': '"true"',
+                    'e': 'false',
+                    'f': '"false"',
+                    'g': 'var',
+                }
+            )),
+            {
+                'null': None,
+                'nullStr': 'null',
+                'true': True,
+                'trueStr': 'true',
+                'false': False,
+                'falseStr': 'false',
+                'unresolvedVar': 'var'
+            }
+        )
+
+        self.assertDictEqual(
+            json.loads(js_to_json(
+                '''{
+                    'int': a,
+                    'intStr': b,
+                    'float': c,
+                    'floatStr': d,
+                }''',
+                {
+                    'a': '123',
+                    'b': '"123"',
+                    'c': '1.23',
+                    'd': '"1.23"',
+                }
+            )),
+            {
+                'int': 123,
+                'intStr': '123',
+                'float': 1.23,
+                'floatStr': '1.23',
+            }
+        )
+
+        self.assertDictEqual(
+            json.loads(js_to_json(
+                '''{
+                    'object': a,
+                    'objectStr': b,
+                    'array': c,
+                    'arrayStr': d,
+                }''',
+                {
+                    'a': '{}',
+                    'b': '"{}"',
+                    'c': '[]',
+                    'd': '"[]"',
+                }
+            )),
+            {
+                'object': {},
+                'objectStr': '{}',
+                'array': [],
+                'arrayStr': '[]',
+            }
+        )
+
     def test_js_to_json_realworld(self):
         inp = '''{
             'clip':{'provider':'pseudo'}
@@ -975,10 +1054,10 @@ class TestUtil(unittest.TestCase):
             !42: 42
         }''')
         self.assertEqual(json.loads(on), {
-            'a': 0,
-            'b': 1,
-            'c': 0,
-            'd': 42.42,
+            'a': True,
+            'b': False,
+            'c': False,
+            'd': True,
             'e': [],
             'f': "abc",
             'g': "",
@@ -1048,10 +1127,26 @@ class TestUtil(unittest.TestCase):
         on = js_to_json('{ "040": "040" }')
         self.assertEqual(json.loads(on), {'040': '040'})
 
+        on = js_to_json('[1,//{},\n2]')
+        self.assertEqual(json.loads(on), [1, 2])
+
+        on = js_to_json(r'"\^\$\#"')
+        self.assertEqual(json.loads(on), R'^$#', msg='Unnecessary escapes should be stripped')
+
+        on = js_to_json('\'"\\""\'')
+        self.assertEqual(json.loads(on), '"""', msg='Unnecessary quote escape should be escaped')
+
     def test_js_to_json_malformed(self):
         self.assertEqual(js_to_json('42a1'), '42"a1"')
         self.assertEqual(js_to_json('42a-1'), '42"a"-1')
 
+    def test_js_to_json_template_literal(self):
+        self.assertEqual(js_to_json('`Hello ${name}`', {'name': '"world"'}), '"Hello world"')
+        self.assertEqual(js_to_json('`${name}${name}`', {'name': '"X"'}), '"XX"')
+        self.assertEqual(js_to_json('`${name}${name}`', {'name': '5'}), '"55"')
+        self.assertEqual(js_to_json('`${name}"${name}"`', {'name': '5'}), '"5\\"5\\""')
+        self.assertEqual(js_to_json('`${name}`', {}), '"name"')
+
     def test_extract_attributes(self):
         self.assertEqual(extract_attributes('<e x="y">'), {'x': 'y'})
         self.assertEqual(extract_attributes("<e x='y'>"), {'x': 'y'})
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b77a7fb0e..b05f65283 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -4365,46 +4365,108 @@ def strip_jsonp(code):
         r'\g<callback_data>', code)
 
 
-def js_to_json(code):
-    COMMENT_RE = r'/\*(?:(?!\*/).)*?\*/|//[^\n]*'
+def js_to_json(code, *args, **kwargs):
+
+    # vars is a dict of (var, val) pairs to substitute
+    vars = args[0] if len(args) > 0 else kwargs.get('vars', {})
+    strict = kwargs.get('strict', False)
+
+    STRING_QUOTES = '\'"`'
+    STRING_RE = '|'.join(r'{0}(?:\\.|[^\\{0}])*{0}'.format(q) for q in STRING_QUOTES)
+    COMMENT_RE = r'/\*(?:(?!\*/).)*?\*/|//[^\n]*\n'
     SKIP_RE = r'\s*(?:{comment})?\s*'.format(comment=COMMENT_RE)
     INTEGER_TABLE = (
         (r'(?s)^(0[xX][0-9a-fA-F]+){skip}:?$'.format(skip=SKIP_RE), 16),
         (r'(?s)^(0+[0-7]+){skip}:?$'.format(skip=SKIP_RE), 8),
+        (r'(?s)^(\d+){skip}:?$'.format(skip=SKIP_RE), 10),
     )
+    # compat candidate
+    JSONDecodeError = json.JSONDecodeError if 'JSONDecodeError' in dir(json) else ValueError
+
+    def process_escape(match):
+        JSON_PASSTHROUGH_ESCAPES = r'"\bfnrtu'
+        escape = match.group(1) or match.group(2)
+
+        return ('\\' + escape if escape in JSON_PASSTHROUGH_ESCAPES
+                else '\\u00' if escape == 'x'
+                else '' if escape == '\n'
+                else escape)
+
+    def template_substitute(match):
+        evaluated = js_to_json(match.group(1), vars, strict=strict)
+        if evaluated[0] == '"':
+            return json.loads(evaluated)
+        return evaluated
 
     def fix_kv(m):
         v = m.group(0)
         if v in ('true', 'false', 'null'):
             return v
-        elif v.startswith('/*') or v.startswith('//') or v.startswith('!') or v == ',':
-            return ""
+        elif v in ('undefined', 'void 0'):
+            return 'null'
+        elif v.startswith('/*') or v.startswith('//') or v == ',':
+            return ''
 
-        if v[0] in ("'", '"'):
-            v = re.sub(r'(?s)\\.|"', lambda m: {
-                '"': '\\"',
-                "\\'": "'",
-                '\\\n': '',
-                '\\x': '\\u00',
-            }.get(m.group(0), m.group(0)), v[1:-1])
-        else:
-            for regex, base in INTEGER_TABLE:
-                im = re.match(regex, v)
-                if im:
-                    i = int(im.group(1), base)
-                    return '"%d":' % i if v.endswith(':') else '%d' % i
+        if v[0] in STRING_QUOTES:
+            v = re.sub(r'(?s)\${([^}]+)}', template_substitute, v[1:-1]) if v[0] == '`' else v[1:-1]
+            escaped = re.sub(r'(?s)(")|\\(.)', process_escape, v)
+            return '"{0}"'.format(escaped)
 
-        return '"%s"' % v
+        inv = IDENTITY
+        im = re.split(r'^!+', v)
+        if len(im) > 1 and not im[-1].endswith(':'):
+            if (len(v) - len(im[1])) % 2 == 1:
+                inv = lambda x: 'true' if x == 0 else 'false'
+            else:
+                inv = lambda x: 'false' if x == 0 else 'true'
+        if not any(x for x in im):
+            return
+        v = im[-1]
+
+        for regex, base in INTEGER_TABLE:
+            im = re.match(regex, v)
+            if im:
+                i = int(im.group(1), base)
+                return ('"%s":' if v.endswith(':') else '%s') % inv(i)
+
+        if v in vars:
+            try:
+                if not strict:
+                    json.loads(vars[v])
+            except JSONDecodeError:
+                return inv(json.dumps(vars[v]))
+            else:
+                return inv(vars[v])
+
+        if not strict:
+            v = try_call(inv, args=(v,), default=v)
+            if v in ('true', 'false'):
+                return v
+            return '"{0}"'.format(v)
+
+        raise ValueError('Unknown value: ' + v)
+
+    def create_map(mobj):
+        return json.dumps(dict(json.loads(js_to_json(mobj.group(1) or '[]', vars=vars))))
+
+    code = re.sub(r'new Map\((\[.*?\])?\)', create_map, code)
+    if not strict:
+        code = re.sub(r'new Date\((".+")\)', r'\g<1>', code)
+        code = re.sub(r'new \w+\((.*?)\)', lambda m: json.dumps(m.group(0)), code)
+        code = re.sub(r'parseInt\([^\d]+(\d+)[^\d]+\)', r'\1', code)
+        code = re.sub(r'\(function\([^)]*\)\s*\{[^}]*\}\s*\)\s*\(\s*(["\'][^)]*["\'])\s*\)', r'\1', code)
 
     return re.sub(r'''(?sx)
-        "(?:[^"\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^"\\]*"|
-        '(?:[^'\\]*(?:\\\\|\\['"nurtbfx/\n]))*[^'\\]*'|
-        {comment}|,(?={skip}[\]}}])|
-        (?:(?<![0-9])[eE]|[a-df-zA-DF-Z_])[.a-zA-Z_0-9]*|
-        \b(?:0[xX][0-9a-fA-F]+|0+[0-7]+)(?:{skip}:)?|
-        [0-9]+(?={skip}:)|
+        {str_}|
+        {comment}|
+        ,(?={skip}[\]}}])|
+        void\s0|
+        !*(?:(?<!\d)[eE]|[a-df-zA-DF-Z_$])[.a-zA-Z_$0-9]*|
+        (?:\b|!+)0(?:[xX][\da-fA-F]+|[0-7]+)(?:{skip}:)?|
+        !+\d+(?:\.\d*)?(?:{skip}:)?|
+        [0-9]+(?:{skip}:)|
         !+
-        '''.format(comment=COMMENT_RE, skip=SKIP_RE), fix_kv, code)
+        '''.format(comment=COMMENT_RE, skip=SKIP_RE, str_=STRING_RE), fix_kv, code)
 
 
 def qualities(quality_ids):

From cb9366eda584fde2421140adf994eadc5bb6b943 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 3 May 2023 10:54:52 +0100
Subject: [PATCH 675/860] [utils] Minor updates (merge_dicts, T)

A couple of mods to ease yt-dlp back-ports:
* add kwargs to merge_dicts:
  `unblank=True` (disallow empty string), `rev=False` (reverse the merge list)
* add `T(x)` shortcut for `{x}`, unsupported in Py2.6
---
 youtube_dl/utils.py | 37 ++++++++++++++++++++++++++++++-------
 1 file changed, 30 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b05f65283..0cbbec0f3 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -4269,7 +4269,8 @@ def variadic(x, allowed_types=NO_DEFAULT):
 
 def dict_get(d, key_or_keys, default=None, skip_false_values=True):
     exp = (lambda x: x or None) if skip_false_values else IDENTITY
-    return traverse_obj(d, *variadic(key_or_keys), expected_type=exp, default=default)
+    return traverse_obj(d, *variadic(key_or_keys), expected_type=exp,
+                        default=default, get_all=False)
 
 
 def try_call(*funcs, **kwargs):
@@ -4302,16 +4303,38 @@ def try_get(src, getter, expected_type=None):
                 return v
 
 
-def merge_dicts(*dicts):
+def merge_dicts(*dicts, **kwargs):
+    """
+        Merge the `dict`s in `dicts` using the first valid value for each key.
+        Normally valid: not None and not an empty string
+
+        Keyword-only args:
+        unblank:    allow empty string if False (default True)
+        rev:        merge dicts in reverse order (default False)
+
+        merge_dicts(dct1, dct2, ..., unblank=False, rev=True)
+        matches {**dct1, **dct2, ...}
+
+        However, merge_dicts(dct1, dct2, ..., rev=True) may often be better.
+    """
+
+    unblank = kwargs.get('unblank', True)
+    rev = kwargs.get('rev', False)
+
+    if unblank:
+        def can_merge_str(k, v, to_dict):
+            return (isinstance(v, compat_str) and v
+                    and isinstance(to_dict[k], compat_str)
+                    and not to_dict[k])
+    else:
+        can_merge_str = lambda k, v, to_dict: False
+
     merged = {}
-    for a_dict in dicts:
+    for a_dict in reversed(dicts) if rev else dicts:
         for k, v in a_dict.items():
             if v is None:
                 continue
-            if (k not in merged
-                    or (isinstance(v, compat_str) and v
-                        and isinstance(merged[k], compat_str)
-                        and not merged[k])):
+            if (k not in merged) or can_merge_str(k, v, merged):
                 merged[k] = v
     return merged
 

From 1e8ccdd2eb77901e18feb8a9d48e62d11651cd1e Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 3 May 2023 13:08:58 +0100
Subject: [PATCH 676/860] [InfoExtractor] Support groups in _`search_regex()`,
 etc

---
 youtube_dl/extractor/common.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 7244e5df6..dbdf456f5 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1005,6 +1005,8 @@ class InfoExtractor(object):
             if group is None:
                 # return the first matching group
                 return next(g for g in mobj.groups() if g is not None)
+            elif isinstance(group, (list, tuple)):
+                return tuple(mobj.group(g) for g in group)
             else:
                 return mobj.group(group)
         elif default is not NO_DEFAULT:
@@ -1020,10 +1022,9 @@ class InfoExtractor(object):
         Like _search_regex, but strips HTML tags and unescapes entities.
         """
         res = self._search_regex(pattern, string, name, default, fatal, flags, group)
-        if res:
-            return clean_html(res).strip()
-        else:
-            return res
+        if isinstance(res, tuple):
+            return tuple(map(clean_html, res))
+        return clean_html(res)
 
     def _get_netrc_login_info(self, netrc_machine=None):
         username = None

From 4566e6e53ebd87c6c548a8414ab5bd742c14c2b0 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 3 May 2023 10:02:25 +0100
Subject: [PATCH 677/860] [GlobalPlayer] Add site extractors back-ported from
 yt-dlp

* from https://github.com/yt-dlp/yt-dlp/pull/6903, thanks garret1317
---
 youtube_dl/extractor/extractors.py   |  15 +-
 youtube_dl/extractor/globalplayer.py | 285 +++++++++++++++++++++++++++
 2 files changed, 296 insertions(+), 4 deletions(-)
 create mode 100644 youtube_dl/extractor/globalplayer.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 3a87f9e33..811a2605a 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -444,6 +444,13 @@ from .gfycat import GfycatIE
 from .giantbomb import GiantBombIE
 from .giga import GigaIE
 from .glide import GlideIE
+from .globalplayer import (
+    GlobalPlayerLiveIE,
+    GlobalPlayerLivePlaylistIE,
+    GlobalPlayerAudioIE,
+    GlobalPlayerAudioEpisodeIE,
+    GlobalPlayerVideoIE
+)
 from .globo import (
     GloboIE,
     GloboArticleIE,
@@ -975,6 +982,10 @@ from .pornhub import (
 from .pornotube import PornotubeIE
 from .pornovoisines import PornoVoisinesIE
 from .pornoxo import PornoXOIE
+from .pr0gramm import (
+    Pr0grammIE,
+    Pr0grammStaticIE,
+)
 from .puhutv import (
     PuhuTVIE,
     PuhuTVSerieIE,
@@ -1678,7 +1689,3 @@ from .zingmp3 import (
 )
 from .zoom import ZoomIE
 from .zype import ZypeIE
-from .pr0gramm import (
-    Pr0grammIE,
-    Pr0grammStaticIE,
-)
diff --git a/youtube_dl/extractor/globalplayer.py b/youtube_dl/extractor/globalplayer.py
new file mode 100644
index 000000000..cceab9e6a
--- /dev/null
+++ b/youtube_dl/extractor/globalplayer.py
@@ -0,0 +1,285 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    clean_html,
+    join_nonempty,
+    merge_dicts,
+    parse_duration,
+    str_or_none,
+    T,
+    traverse_obj,
+    unified_strdate,
+    unified_timestamp,
+    urlhandle_detect_ext,
+)
+
+
+class GlobalPlayerBaseIE(InfoExtractor):
+
+    import re
+
+    @classmethod
+    def _match_valid_url(cls, url):
+        return cls.re.match(cls._VALID_URL, url)
+
+    def _search_nextjs_data(self, webpage, video_id, **kw):
+        return self._parse_json(
+            self._search_regex(
+                r'(?s)<script[^>]+id=[\'"]__NEXT_DATA__[\'"][^>]*>([^<]+)</script>',
+                webpage, 'next.js data', **kw),
+            video_id, **kw)
+
+    def _get_page_props(self, url, video_id):
+        webpage = self._download_webpage(url, video_id)
+        return self._search_nextjs_data(webpage, video_id)['props']['pageProps']
+
+    def _request_ext(self, url, video_id):
+        return urlhandle_detect_ext(self._request_webpage(  # Server rejects HEAD requests
+            url, video_id, note='Determining source extension'))
+
+    def _extract_audio(self, episode, series):
+
+        def clean_desc(x):
+            x = clean_html(x)
+            if x:
+                x = x.replace('\xa0', ' ')
+            return x
+
+        return merge_dicts({
+            'vcodec': 'none',
+        }, traverse_obj(series, {
+            'series': 'title',
+            'series_id': 'id',
+            'thumbnail': 'imageUrl',
+            'uploader': 'itunesAuthor',  # podcasts only
+        }), traverse_obj(episode, {
+            'id': 'id',
+            'description': ('description', T(clean_desc)),
+            'duration': ('duration', T(parse_duration)),
+            'thumbnail': 'imageUrl',
+            'url': 'streamUrl',
+            'timestamp': (('pubDate', 'startDate'), T(unified_timestamp)),
+            'title': 'title',
+        }, get_all=False), rev=True)
+
+
+class GlobalPlayerLiveIE(GlobalPlayerBaseIE):
+    _VALID_URL = r'https?://www\.globalplayer\.com/live/(?P<id>\w+)/\w+'
+    _TESTS = [{
+        'url': 'https://www.globalplayer.com/live/smoothchill/uk/',
+        'info_dict': {
+            'id': '2mx1E',
+            'ext': 'aac',
+            'display_id': 'smoothchill-uk',
+            'title': 're:^Smooth Chill.+$',
+            'thumbnail': 'https://herald.musicradio.com/media/f296ade8-50c9-4f60-911f-924e96873620.png',
+            'description': 'Music To Chill To',
+            # 'live_status': 'is_live',
+            'is_live': True,
+        },
+    }, {
+        # national station
+        'url': 'https://www.globalplayer.com/live/heart/uk/',
+        'info_dict': {
+            'id': '2mwx4',
+            'ext': 'aac',
+            'description': 'turn up the feel good!',
+            'thumbnail': 'https://herald.musicradio.com/media/49b9e8cb-15bf-4bf2-8c28-a4850cc6b0f3.png',
+            # 'live_status': 'is_live',
+            'is_live': True,
+            'title': 're:^Heart UK.+$',
+            'display_id': 'heart-uk',
+        },
+    }, {
+        # regional variation
+        'url': 'https://www.globalplayer.com/live/heart/london/',
+        'info_dict': {
+            'id': 'AMqg',
+            'ext': 'aac',
+            'thumbnail': 'https://herald.musicradio.com/media/49b9e8cb-15bf-4bf2-8c28-a4850cc6b0f3.png',
+            'title': 're:^Heart London.+$',
+            # 'live_status': 'is_live',
+            'is_live': True,
+            'display_id': 'heart-london',
+            'description': 'turn up the feel good!',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        station = self._get_page_props(url, video_id)['station']
+        stream_url = station['streamUrl']
+
+        return merge_dicts({
+            'id': station['id'],
+            'display_id': (
+                join_nonempty('brandSlug', 'slug', from_dict=station)
+                or station.get('legacyStationPrefix')),
+            'url': stream_url,
+            'ext': self._request_ext(stream_url, video_id),
+            'vcodec': 'none',
+            'is_live': True,
+        }, {
+            'title': self._live_title(traverse_obj(
+                station, (('name', 'brandName'), T(str_or_none)),
+                get_all=False)),
+        }, traverse_obj(station, {
+            'description': 'tagline',
+            'thumbnail': 'brandLogo',
+        }), rev=True)
+
+
+class GlobalPlayerLivePlaylistIE(GlobalPlayerBaseIE):
+    _VALID_URL = r'https?://www\.globalplayer\.com/playlists/(?P<id>\w+)'
+    _TESTS = [{
+        # "live playlist"
+        'url': 'https://www.globalplayer.com/playlists/8bLk/',
+        'info_dict': {
+            'id': '8bLk',
+            'ext': 'aac',
+            # 'live_status': 'is_live',
+            'is_live': True,
+            'description': 'md5:e10f5e10b01a7f2c14ba815509fbb38d',
+            'thumbnail': 'https://images.globalplayer.com/images/551379?width=450&signature=oMLPZIoi5_dBSHnTMREW0Xg76mA=',
+            'title': 're:^Classic FM Hall of Fame.+$'
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        station = self._get_page_props(url, video_id)['playlistData']
+        stream_url = station['streamUrl']
+
+        return merge_dicts({
+            'id': video_id,
+            'url': stream_url,
+            'ext': self._request_ext(stream_url, video_id),
+            'vcodec': 'none',
+            'is_live': True,
+        }, traverse_obj(station, {
+            'title': 'title',
+            'description': 'description',
+            'thumbnail': 'image',
+        }), rev=True)
+
+
+class GlobalPlayerAudioIE(GlobalPlayerBaseIE):
+    _VALID_URL = r'https?://www\.globalplayer\.com/(?:(?P<podcast>podcasts)/|catchup/\w+/\w+/)(?P<id>\w+)/?(?:$|[?#])'
+    _TESTS = [{
+        # podcast
+        'url': 'https://www.globalplayer.com/podcasts/42KuaM/',
+        'playlist_mincount': 5,
+        'info_dict': {
+            'id': '42KuaM',
+            'title': 'Filthy Ritual',
+            'thumbnail': 'md5:60286e7d12d795bd1bbc9efc6cee643e',
+            'categories': ['Society & Culture', 'True Crime'],
+            'uploader': 'Global',
+            'description': 'md5:da5b918eac9ae319454a10a563afacf9',
+        },
+    }, {
+        # radio catchup
+        'url': 'https://www.globalplayer.com/catchup/lbc/uk/46vyD7z/',
+        'playlist_mincount': 2,
+        'info_dict': {
+            'id': '46vyD7z',
+            'description': 'Nick Ferrari At Breakfast is Leading Britain\'s Conversation.',
+            'title': 'Nick Ferrari',
+            'thumbnail': 'md5:4df24d8a226f5b2508efbcc6ae874ebf',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id, podcast = self._match_valid_url(url).group('id', 'podcast')
+        props = self._get_page_props(url, video_id)
+        series = props['podcastInfo'] if podcast else props['catchupInfo']
+
+        return merge_dicts({
+            '_type': 'playlist',
+            'id': video_id,
+            'entries': [self._extract_audio(ep, series) for ep in traverse_obj(
+                        series, ('episodes', lambda _, v: v['id'] and v['streamUrl']))],
+            'categories': traverse_obj(series, ('categories', Ellipsis, 'name')) or None,
+        }, traverse_obj(series, {
+            'description': 'description',
+            'thumbnail': 'imageUrl',
+            'title': 'title',
+            'uploader': 'itunesAuthor',  # podcasts only
+        }), rev=True)
+
+
+class GlobalPlayerAudioEpisodeIE(GlobalPlayerBaseIE):
+    _VALID_URL = r'https?://www\.globalplayer\.com/(?:(?P<podcast>podcasts)|catchup/\w+/\w+)/episodes/(?P<id>\w+)/?(?:$|[?#])'
+    _TESTS = [{
+        # podcast
+        'url': 'https://www.globalplayer.com/podcasts/episodes/7DrfNnE/',
+        'info_dict': {
+            'id': '7DrfNnE',
+            'ext': 'mp3',
+            'title': 'Filthy Ritual - Trailer',
+            'description': 'md5:1f1562fd0f01b4773b590984f94223e0',
+            'thumbnail': 'md5:60286e7d12d795bd1bbc9efc6cee643e',
+            'duration': 225.0,
+            'timestamp': 1681254900,
+            'series': 'Filthy Ritual',
+            'series_id': '42KuaM',
+            'upload_date': '20230411',
+            'uploader': 'Global',
+        },
+    }, {
+        # radio catchup
+        'url': 'https://www.globalplayer.com/catchup/lbc/uk/episodes/2zGq26Vcv1fCWhddC4JAwETXWe/',
+        'only_matching': True,
+        # expired: refresh the details with a current show for a full test
+        'info_dict': {
+            'id': '2zGq26Vcv1fCWhddC4JAwETXWe',
+            'ext': 'm4a',
+            'timestamp': 1682056800,
+            'series': 'Nick Ferrari',
+            'thumbnail': 'md5:4df24d8a226f5b2508efbcc6ae874ebf',
+            'upload_date': '20230421',
+            'series_id': '46vyD7z',
+            'description': 'Nick Ferrari At Breakfast is Leading Britain\'s Conversation.',
+            'title': 'Nick Ferrari',
+            'duration': 10800.0,
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id, podcast = self._match_valid_url(url).group('id', 'podcast')
+        props = self._get_page_props(url, video_id)
+        episode = props['podcastEpisode'] if podcast else props['catchupEpisode']
+
+        return self._extract_audio(
+            episode, traverse_obj(episode, 'podcast', 'show', expected_type=dict) or {})
+
+
+class GlobalPlayerVideoIE(GlobalPlayerBaseIE):
+    _VALID_URL = r'https?://www\.globalplayer\.com/videos/(?P<id>\w+)'
+    _TESTS = [{
+        'url': 'https://www.globalplayer.com/videos/2JsSZ7Gm2uP/',
+        'info_dict': {
+            'id': '2JsSZ7Gm2uP',
+            'ext': 'mp4',
+            'description': 'md5:6a9f063c67c42f218e42eee7d0298bfd',
+            'thumbnail': 'md5:d4498af48e15aae4839ce77b97d39550',
+            'upload_date': '20230420',
+            'title': 'Treble Malakai Bayoh sings a sublime Handel aria at Classic FM Live',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        meta = self._get_page_props(url, video_id)['videoData']
+
+        return merge_dicts({
+            'id': video_id,
+        }, traverse_obj(meta, {
+            'url': 'url',
+            'thumbnail': ('image', 'url'),
+            'title': 'title',
+            'upload_date': ('publish_date', T(unified_strdate)),
+            'description': 'description',
+        }), rev=True)

From eaaf4c6736b98e20a1923162ae05952c8cb51ee1 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 3 May 2023 10:04:46 +0100
Subject: [PATCH 678/860] [Whyp] Add extractor back-ported from yt-dlp

* from https://github.com/yt-dlp/yt-dlp/pull/6803, thanks CoryTibbettsDev
---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/whyp.py       | 78 ++++++++++++++++++++++++++++++
 2 files changed, 79 insertions(+)
 create mode 100644 youtube_dl/extractor/whyp.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 811a2605a..9f247dbbf 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1576,6 +1576,7 @@ from .weibo import (
     WeiboMobileIE
 )
 from .weiqitv import WeiqiTVIE
+from .whyp import WhypIE
 from .wistia import (
     WistiaIE,
     WistiaPlaylistIE,
diff --git a/youtube_dl/extractor/whyp.py b/youtube_dl/extractor/whyp.py
new file mode 100644
index 000000000..16f9154ad
--- /dev/null
+++ b/youtube_dl/extractor/whyp.py
@@ -0,0 +1,78 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    merge_dicts,
+    str_or_none,
+    T,
+    traverse_obj,
+    url_or_none,
+)
+
+
+class WhypIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?whyp\.it/tracks/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.whyp.it/tracks/18337/home-page-example-track-b4kq7',
+        'md5': 'c1187b42ebf8605284e3dc92aeb33d16',
+        'info_dict': {
+            'url': 'https://cdn.whyp.it/50eb17cc-e9ff-4e18-b89b-dc9206a95cb1.mp3',
+            'id': '18337',
+            'title': 'Home Page Example Track',
+            'description': 'md5:bd758000fb93f3159339c852b5b9133c',
+            'ext': 'mp3',
+            'duration': 52.82,
+            'uploader': 'Brad',
+            'uploader_id': '1',
+            'thumbnail': 'https://cdn.whyp.it/a537bb36-3373-4c61-96c8-27fc1b2f427a.jpg',
+        },
+    }, {
+        'url': 'https://www.whyp.it/tracks/18337',
+        'only_matching': True,
+    }]
+
+    def _search_nuxt_data(self, webpage, video_id, context_name='__NUXT__', fatal=True, traverse=('data', 0)):
+        """Parses Nuxt.js metadata. This works as long as the function __NUXT__ invokes is a pure function"""
+
+        import functools
+        import json
+        import re
+        from ..utils import (js_to_json, NO_DEFAULT)
+
+        re_ctx = re.escape(context_name)
+        FUNCTION_RE = r'\(function\((?P<arg_keys>.*?)\){return\s+(?P<js>{.*?})\s*;?\s*}\((?P<arg_vals>.*?)\)'
+        js, arg_keys, arg_vals = self._search_regex(
+            (p.format(re_ctx, FUNCTION_RE) for p in (r'<script>\s*window\.{0}={1}\s*\)\s*;?\s*</script>', r'{0}\(.*?{1}')),
+            webpage, context_name, group=('js', 'arg_keys', 'arg_vals'),
+            default=NO_DEFAULT if fatal else (None, None, None))
+        if js is None:
+            return {}
+
+        args = dict(zip(arg_keys.split(','), map(json.dumps, self._parse_json(
+            '[{0}]'.format(arg_vals), video_id, transform_source=js_to_json, fatal=fatal) or ())))
+
+        ret = self._parse_json(js, video_id, transform_source=functools.partial(js_to_json, vars=args), fatal=fatal)
+        return traverse_obj(ret, traverse) or {}
+
+    def _real_extract(self, url):
+        unique_id = self._match_id(url)
+        webpage = self._download_webpage(url, unique_id)
+        data = self._search_nuxt_data(webpage, unique_id)['rawTrack']
+
+        return merge_dicts({
+            'url': data['audio_url'],
+            'id': unique_id,
+        }, traverse_obj(data, {
+            'title': 'title',
+            'description': 'description',
+            'duration': ('duration', T(float_or_none)),
+            'uploader': ('user', 'username'),
+            'uploader_id': ('user', 'id', T(str_or_none)),
+            'thumbnail': ('artwork_url', T(url_or_none)),
+        }), {
+            'ext': 'mp3',
+            'vcodec': 'none',
+            'http_headers': {'Referer': 'https://whyp.it/'},
+        }, rev=True)

From 4339910df3fe97a054069cb98da594dd1b50c13a Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 3 May 2023 10:07:35 +0100
Subject: [PATCH 679/860] [DLF] Add site extractors back-ported from yt-dlp

* from https://github.com/yt-dlp/yt-dlp/pull/6697, thanks nick-cd
---
 youtube_dl/extractor/dlf.py        | 204 +++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |   4 +
 2 files changed, 208 insertions(+)
 create mode 100644 youtube_dl/extractor/dlf.py

diff --git a/youtube_dl/extractor/dlf.py b/youtube_dl/extractor/dlf.py
new file mode 100644
index 000000000..cc3de4582
--- /dev/null
+++ b/youtube_dl/extractor/dlf.py
@@ -0,0 +1,204 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import re
+
+from .common import InfoExtractor
+from ..compat import (
+    compat_str,
+)
+from ..utils import (
+    determine_ext,
+    extract_attributes,
+    int_or_none,
+    merge_dicts,
+    traverse_obj,
+    url_or_none,
+    variadic,
+)
+
+
+class DLFBaseIE(InfoExtractor):
+    _VALID_URL_BASE = r'https?://(?:www\.)?deutschlandfunk\.de/'
+    _BUTTON_REGEX = r'(<button[^>]+alt="Anhören"[^>]+data-audio-diraid[^>]*>)'
+
+    def _parse_button_attrs(self, button, audio_id=None):
+        attrs = extract_attributes(button)
+        audio_id = audio_id or attrs['data-audio-diraid']
+
+        url = traverse_obj(
+            attrs, 'data-audio-download-src', 'data-audio', 'data-audioreference',
+            'data-audio-src', expected_type=url_or_none)
+        ext = determine_ext(url)
+        formats = (self._extract_m3u8_formats(url, audio_id, fatal=False)
+                   if ext == 'm3u8' else [{'url': url, 'ext': ext, 'vcodec': 'none'}])
+        self._sort_formats(formats)
+
+        def traverse_attrs(path):
+            path = list(variadic(path))
+            t = path.pop() if callable(path[-1]) else None
+            return traverse_obj(attrs, path, expected_type=t, get_all=False)
+
+        def txt_or_none(v, default=None):
+            return default if v is None else (compat_str(v).strip() or default)
+
+        return merge_dicts(*reversed([{
+            'id': audio_id,
+            # 'extractor_key': DLFIE.ie_key(),
+            # 'extractor': DLFIE.IE_NAME,
+            'formats': formats,
+        }, dict((k, traverse_attrs(v)) for k, v in {
+            'title': (('data-audiotitle', 'data-audio-title', 'data-audio-download-tracking-title'), txt_or_none),
+            'duration': (('data-audioduration', 'data-audio-duration'), int_or_none),
+            'thumbnail': ('data-audioimage', url_or_none),
+            'uploader': 'data-audio-producer',
+            'series': 'data-audio-series',
+            'channel': 'data-audio-origin-site-name',
+            'webpage_url': ('data-audio-download-tracking-path', url_or_none),
+        }.items())]))
+
+
+class DLFIE(DLFBaseIE):
+    IE_NAME = 'dlf'
+    _VALID_URL = DLFBaseIE._VALID_URL_BASE + r'[\w-]+-dlf-(?P<id>[\da-f]{8})-100\.html'
+    _TESTS = [
+        # Audio as an HLS stream
+        {
+            'url': 'https://www.deutschlandfunk.de/tanz-der-saiteninstrumente-das-wild-strings-trio-aus-slowenien-dlf-03a3eb19-100.html',
+            'info_dict': {
+                'id': '03a3eb19',
+                'title': r're:Tanz der Saiteninstrumente [-/] Das Wild Strings Trio aus Slowenien',
+                'ext': 'm4a',
+                'duration': 3298,
+                'thumbnail': 'https://assets.deutschlandfunk.de/FALLBACK-IMAGE-AUDIO/512x512.png?t=1603714364673',
+                'uploader': 'Deutschlandfunk',
+                'series': 'On Stage',
+                'channel': 'deutschlandfunk'
+            },
+            'params': {
+                'skip_download': 'm3u8'
+            },
+            'skip': 'This webpage no longer exists'
+        }, {
+            'url': 'https://www.deutschlandfunk.de/russische-athleten-kehren-zurueck-auf-die-sportbuehne-ein-gefaehrlicher-tueroeffner-dlf-d9cc1856-100.html',
+            'info_dict': {
+                'id': 'd9cc1856',
+                'title': 'Russische Athleten kehren zurück auf die Sportbühne: Ein gefährlicher Türöffner',
+                'ext': 'mp3',
+                'duration': 291,
+                'thumbnail': 'https://assets.deutschlandfunk.de/FALLBACK-IMAGE-AUDIO/512x512.png?t=1603714364673',
+                'uploader': 'Deutschlandfunk',
+                'series': 'Kommentare und Themen der Woche',
+                'channel': 'deutschlandfunk'
+            }
+        },
+    ]
+
+    def _real_extract(self, url):
+        audio_id = self._match_id(url)
+        webpage = self._download_webpage(url, audio_id)
+
+        return self._parse_button_attrs(
+            self._search_regex(self._BUTTON_REGEX, webpage, 'button'), audio_id)
+
+
+class DLFCorpusIE(DLFBaseIE):
+    IE_NAME = 'dlf:corpus'
+    IE_DESC = 'DLF Multi-feed Archives'
+    _VALID_URL = DLFBaseIE._VALID_URL_BASE + r'(?P<id>(?![\w-]+-dlf-[\da-f]{8})[\w-]+-\d+)\.html'
+    _TESTS = [
+        # Recorded news broadcast with referrals to related broadcasts
+        {
+            'url': 'https://www.deutschlandfunk.de/fechten-russland-belarus-ukraine-protest-100.html',
+            'info_dict': {
+                'id': 'fechten-russland-belarus-ukraine-protest-100',
+                'title': r're:Wiederzulassung als neutrale Athleten [-/] Was die Rückkehr russischer und belarussischer Sportler beim Fechten bedeutet',
+                'description': 'md5:91340aab29c71aa7518ad5be13d1e8ad'
+            },
+            'playlist_mincount': 5,
+            'playlist': [{
+                'info_dict': {
+                    'id': '1fc5d64a',
+                    'title': r're:Wiederzulassung als neutrale Athleten [-/] Was die Rückkehr russischer und belarussischer Sportler beim Fechten bedeutet',
+                    'ext': 'mp3',
+                    'duration': 252,
+                    'thumbnail': 'https://assets.deutschlandfunk.de/aad16241-6b76-4a09-958b-96d0ee1d6f57/512x512.jpg?t=1679480020313',
+                    'uploader': 'Deutschlandfunk',
+                    'series': 'Sport',
+                    'channel': 'deutschlandfunk'
+                }
+            }, {
+                'info_dict': {
+                    'id': '2ada145f',
+                    'title': r're:(?:Sportpolitik / )?Fechtverband votiert für Rückkehr russischer Athleten',
+                    'ext': 'mp3',
+                    'duration': 336,
+                    'thumbnail': 'https://assets.deutschlandfunk.de/FILE_93982766f7317df30409b8a184ac044a/512x512.jpg?t=1678547581005',
+                    'uploader': 'Deutschlandfunk',
+                    'series': 'Deutschlandfunk Nova',
+                    'channel': 'deutschlandfunk-nova'
+                }
+            }, {
+                'info_dict': {
+                    'id': '5e55e8c9',
+                    'title': r're:Wiederzulassung von Russland und Belarus [-/] "Herumlavieren" des Fechter-Bundes sorgt für Unverständnis',
+                    'ext': 'mp3',
+                    'duration': 187,
+                    'thumbnail': 'https://assets.deutschlandfunk.de/a595989d-1ed1-4a2e-8370-b64d7f11d757/512x512.jpg?t=1679173825412',
+                    'uploader': 'Deutschlandfunk',
+                    'series': 'Sport am Samstag',
+                    'channel': 'deutschlandfunk'
+                }
+            }, {
+                'info_dict': {
+                    'id': '47e1a096',
+                    'title': r're:Rückkehr Russlands im Fechten [-/] "Fassungslos, dass es einfach so passiert ist"',
+                    'ext': 'mp3',
+                    'duration': 602,
+                    'thumbnail': 'https://assets.deutschlandfunk.de/da4c494a-21cc-48b4-9cc7-40e09fd442c2/512x512.jpg?t=1678562155770',
+                    'uploader': 'Deutschlandfunk',
+                    'series': 'Sport am Samstag',
+                    'channel': 'deutschlandfunk'
+                }
+            }, {
+                'info_dict': {
+                    'id': '5e55e8c9',
+                    'title': r're:Wiederzulassung von Russland und Belarus [-/] "Herumlavieren" des Fechter-Bundes sorgt für Unverständnis',
+                    'ext': 'mp3',
+                    'duration': 187,
+                    'thumbnail': 'https://assets.deutschlandfunk.de/a595989d-1ed1-4a2e-8370-b64d7f11d757/512x512.jpg?t=1679173825412',
+                    'uploader': 'Deutschlandfunk',
+                    'series': 'Sport am Samstag',
+                    'channel': 'deutschlandfunk'
+                }
+            }]
+        },
+        # Podcast feed with tag buttons, playlist count fluctuates
+        {
+            'url': 'https://www.deutschlandfunk.de/kommentare-und-themen-der-woche-100.html',
+            'info_dict': {
+                'id': 'kommentare-und-themen-der-woche-100',
+                'title': 'Meinung - Kommentare und Themen der Woche',
+                'description': 'md5:2901bbd65cd2d45e116d399a099ce5d5',
+            },
+            'playlist_mincount': 10,
+        },
+        # Podcast feed with no description
+        {
+            'url': 'https://www.deutschlandfunk.de/podcast-tolle-idee-100.html',
+            'info_dict': {
+                'id': 'podcast-tolle-idee-100',
+                'title': 'Wissenschaftspodcast - Tolle Idee! - Was wurde daraus?',
+            },
+            'playlist_mincount': 11,
+        },
+    ]
+
+    def _real_extract(self, url):
+        playlist_id = self._match_id(url)
+        webpage = self._download_webpage(url, playlist_id)
+
+        return self.playlist_result(
+            map(self._parse_button_attrs, re.findall(self._BUTTON_REGEX, webpage)),
+            playlist_id, self._html_search_meta(['og:title', 'twitter:title'], webpage, default=None),
+            self._html_search_meta(['description', 'og:description', 'twitter:description'], webpage, default=None))
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 9f247dbbf..be73c0665 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -295,6 +295,10 @@ from .dbtv import DBTVIE
 from .dctp import DctpTvIE
 from .deezer import DeezerPlaylistIE
 from .democracynow import DemocracynowIE
+from .dlf import (
+    DLFCorpusIE,
+    DLFIE,
+)
 from .dfb import DFBIE
 from .dhm import DHMIE
 from .digg import DiggIE

From 846522204104e3078c597fa1872465024a684ad6 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 4 May 2023 00:08:26 +0100
Subject: [PATCH 680/860] [Clipchamp] Add new extractor back-ported from yt-dlp

---
 youtube_dl/extractor/clipchamp.py  | 76 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 77 insertions(+)
 create mode 100644 youtube_dl/extractor/clipchamp.py

diff --git a/youtube_dl/extractor/clipchamp.py b/youtube_dl/extractor/clipchamp.py
new file mode 100644
index 000000000..5a732e808
--- /dev/null
+++ b/youtube_dl/extractor/clipchamp.py
@@ -0,0 +1,76 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..compat import compat_str
+from ..utils import (
+    ExtractorError,
+    merge_dicts,
+    T,
+    traverse_obj,
+    unified_timestamp,
+    url_or_none,
+)
+
+
+class ClipchampIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?clipchamp\.com/watch/(?P<id>[\w-]+)'
+    _TESTS = [{
+        'url': 'https://clipchamp.com/watch/gRXZ4ZhdDaU',
+        'info_dict': {
+            'id': 'gRXZ4ZhdDaU',
+            'ext': 'mp4',
+            'title': 'Untitled video',
+            'uploader': 'Alexander Schwartz',
+            'timestamp': 1680805580,
+            'upload_date': '20230406',
+            'thumbnail': r're:^https?://.+\.jpg',
+        },
+        'params': {
+            'skip_download': 'm3u8',
+            'format': 'bestvideo',
+        },
+    }]
+
+    _STREAM_URL_TMPL = 'https://%s.cloudflarestream.com/%s/manifest/video.%s'
+    _STREAM_URL_QUERY = {'parentOrigin': 'https://clipchamp.com'}
+
+    def _search_nextjs_data(self, webpage, video_id, **kw):
+        return self._parse_json(
+            self._search_regex(
+                r'(?s)<script[^>]+id=[\'"]__NEXT_DATA__[\'"][^>]*>([^<]+)</script>',
+                webpage, 'next.js data', **kw),
+            video_id, **kw)
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        webpage = self._download_webpage(url, video_id)
+        data = self._search_nextjs_data(webpage, video_id)['props']['pageProps']['video']
+
+        storage_location = data.get('storage_location')
+        if storage_location != 'cf_stream':
+            raise ExtractorError('Unsupported clip storage location "%s"' % (storage_location,))
+
+        path = data['download_url']
+        iframe = self._download_webpage(
+            'https://iframe.cloudflarestream.com/' + path, video_id, 'Downloading player iframe')
+        subdomain = self._search_regex(
+            r'''\bcustomer-domain-prefix\s*=\s*("|')(?P<sd>[\w-]+)\1''', iframe,
+            'subdomain', group='sd', fatal=False) or 'customer-2ut9yn3y6fta1yxe'
+
+        formats = self._extract_mpd_formats(
+            self._STREAM_URL_TMPL % (subdomain, path, 'mpd'), video_id,
+            query=self._STREAM_URL_QUERY, fatal=False, mpd_id='dash')
+        formats.extend(self._extract_m3u8_formats(
+            self._STREAM_URL_TMPL % (subdomain, path, 'm3u8'), video_id, 'mp4',
+            query=self._STREAM_URL_QUERY, fatal=False, m3u8_id='hls'))
+
+        return merge_dicts({
+            'id': video_id,
+            'formats': formats,
+            'uploader': ' '.join(traverse_obj(data, ('creator', ('first_name', 'last_name'), T(compat_str)))) or None,
+        }, traverse_obj(data, {
+            'title': ('project', 'project_name', T(compat_str)),
+            'timestamp': ('created_at', T(unified_timestamp)),
+            'thumbnail': ('thumbnail_url', T(url_or_none)),
+        }), rev=True)
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index be73c0665..42b009ef5 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -226,6 +226,7 @@ from .ciscolive import (
     CiscoLiveSearchIE,
 )
 from .cjsw import CJSWIE
+from .clipchamp import ClipchampIE
 from .cliphunter import CliphunterIE
 from .clippit import ClippitIE
 from .cliprs import ClipRsIE

From b2741f2654e6ddfebc1771b5d5fadb5fd6fe3863 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 5 May 2023 19:25:42 +0100
Subject: [PATCH 681/860] [InfoExtractor] Add search methods for Next/Nuxt.js
 from yt-dlp * add _search_nextjs_data(), from
 https://github.com/yt-dlp/yt-dlp/pull/1386   thanks selfisekai * add
 _search_nuxt_data(), from https://github.com/yt-dlp/yt-dlp/pull/1921,  
 thanks Lesmiscore, pukkandan * add tests for the above * also fix HTML5 type
 recognition and tests, from  
 https://github.com/yt-dlp/yt-dlp/commit/222a230871fe4fe63f35c49590379c9a77116819,
   thanks Lesmiscore * update extractors in PR using above, fix tests.

---
 test/test_InfoExtractor.py           | 111 +++++++++++++++++++++++++--
 youtube_dl/extractor/clipchamp.py    |   7 --
 youtube_dl/extractor/common.py       |  51 +++++++++++-
 youtube_dl/extractor/globalplayer.py |  32 ++++----
 youtube_dl/extractor/whyp.py         |  25 +-----
 5 files changed, 168 insertions(+), 58 deletions(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 6d25441db..34773fbd0 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -7,15 +7,33 @@ import io
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from test.helper import FakeYDL, expect_dict, expect_value, http_server_port
-from youtube_dl.compat import compat_etree_fromstring, compat_http_server
-from youtube_dl.extractor.common import InfoExtractor
-from youtube_dl.extractor import YoutubeIE, get_info_extractor
-from youtube_dl.utils import encode_data_uri, strip_jsonp, ExtractorError, RegexNotFoundError
 import threading
 
+from test.helper import (
+    expect_dict,
+    expect_value,
+    FakeYDL,
+    http_server_port,
+)
+from youtube_dl.compat import (
+    compat_etree_fromstring,
+    compat_http_server,
+)
+from youtube_dl.extractor.common import InfoExtractor
+from youtube_dl.extractor import (
+    get_info_extractor,
+    YoutubeIE,
+)
+from youtube_dl.utils import (
+    encode_data_uri,
+    ExtractorError,
+    RegexNotFoundError,
+    strip_jsonp,
+)
+
 
 TEAPOT_RESPONSE_STATUS = 418
 TEAPOT_RESPONSE_BODY = "<h1>418 I'm a teapot</h1>"
@@ -100,6 +118,71 @@ class TestInfoExtractor(unittest.TestCase):
         self.assertRaises(RegexNotFoundError, ie._html_search_meta, 'z', html, None, fatal=True)
         self.assertRaises(RegexNotFoundError, ie._html_search_meta, ('z', 'x'), html, None, fatal=True)
 
+    def test_search_nextjs_data(self):
+        html = '''
+<!DOCTYPE html>
+<html>
+<head>
+  <meta http-equiv="content-type" content=
+  "text/html; charset=utf-8">
+  <meta name="viewport" content="width=device-width">
+  <title>Test _search_nextjs_data()</title>
+</head>
+<body>
+  <div id="__next">
+    <div style="background-color:#17171E" class="FU" dir="ltr">
+      <div class="sc-93de261d-0 dyzzYE">
+        <div>
+          <header class="HD"></header>
+          <main class="MN">
+            <div style="height:0" class="HT0">
+              <div style="width:NaN%" data-testid=
+              "stream-container" class="WDN"></div>
+            </div>
+          </main>
+        </div>
+        <footer class="sc-6e5faf91-0 dEGaHS"></footer>
+      </div>
+    </div>
+  </div>
+  <script id="__NEXT_DATA__" type="application/json">
+  {"props":{"pageProps":{"video":{"id":"testid"}}}}
+  </script>
+</body>
+</html>
+'''
+        search = self.ie._search_nextjs_data(html, 'testID')
+        self.assertEqual(search['props']['pageProps']['video']['id'], 'testid')
+
+    def test_search_nuxt_data(self):
+        html = '''
+<!DOCTYPE html>
+<html>
+<head>
+  <meta http-equiv="content-type" content=
+  "text/html; charset=utf-8">
+  <title>Nuxt.js Test Page</title>
+  <meta name="viewport" content=
+  "width=device-width, initial-scale=1">
+  <meta data-hid="robots" name="robots" content="all">
+</head>
+<body class="BD">
+  <div id="__layout">
+    <h1 class="H1">Example heading</h1>
+    <div class="IN">
+      <p>Decoy text</p>
+    </div>
+  </div>
+  <script>
+  window.__NUXT__=(function(a,b,c,d,e,f,g,h){return {decoy:" default",data:[{track:{id:f,title:g}}]}}(null,null,"c",null,null,"testid","Nuxt.js title",null));
+  </script>
+  <script src="/_nuxt/a12345b.js" defer="defer"></script>
+</body>
+</html>
+'''
+        search = self.ie._search_nuxt_data(html, 'testID')
+        self.assertEqual(search['track']['id'], 'testid')
+
     def test_search_json_ld_realworld(self):
         # https://github.com/ytdl-org/youtube-dl/issues/23306
         expect_dict(
@@ -348,6 +431,24 @@ class TestInfoExtractor(unittest.TestCase):
                 }],
             })
 
+        # from https://0000.studio/
+        # with type attribute but without extension in URL
+        expect_dict(
+            self,
+            self.ie._parse_html5_media_entries(
+                'https://0000.studio',
+                r'''
+                <video src="https://d1ggyt9m8pwf3g.cloudfront.net/protected/ap-northeast-1:1864af40-28d5-492b-b739-b32314b1a527/archive/clip/838db6a7-8973-4cd6-840d-8517e4093c92"
+                    controls="controls" type="video/mp4" preload="metadata" autoplay="autoplay" playsinline class="object-contain">
+                </video>
+                ''', None)[0],
+            {
+                'formats': [{
+                    'url': 'https://d1ggyt9m8pwf3g.cloudfront.net/protected/ap-northeast-1:1864af40-28d5-492b-b739-b32314b1a527/archive/clip/838db6a7-8973-4cd6-840d-8517e4093c92',
+                    'ext': 'mp4',
+                }],
+            })
+
     def test_extract_jwplayer_data_realworld(self):
         # from http://www.suffolk.edu/sjc/
         expect_dict(
diff --git a/youtube_dl/extractor/clipchamp.py b/youtube_dl/extractor/clipchamp.py
index 5a732e808..3b485eaab 100644
--- a/youtube_dl/extractor/clipchamp.py
+++ b/youtube_dl/extractor/clipchamp.py
@@ -35,13 +35,6 @@ class ClipchampIE(InfoExtractor):
     _STREAM_URL_TMPL = 'https://%s.cloudflarestream.com/%s/manifest/video.%s'
     _STREAM_URL_QUERY = {'parentOrigin': 'https://clipchamp.com'}
 
-    def _search_nextjs_data(self, webpage, video_id, **kw):
-        return self._parse_json(
-            self._search_regex(
-                r'(?s)<script[^>]+id=[\'"]__NEXT_DATA__[\'"][^>]*>([^<]+)</script>',
-                webpage, 'next.js data', **kw),
-            video_id, **kw)
-
     def _real_extract(self, url):
         video_id = self._match_id(url)
         webpage = self._download_webpage(url, video_id)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index dbdf456f5..549781186 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -3,6 +3,7 @@ from __future__ import unicode_literals
 
 import base64
 import datetime
+import functools
 import hashlib
 import json
 import netrc
@@ -23,6 +24,7 @@ from ..compat import (
     compat_getpass,
     compat_integer_types,
     compat_http_client,
+    compat_map as map,
     compat_os_name,
     compat_str,
     compat_urllib_error,
@@ -31,6 +33,7 @@ from ..compat import (
     compat_urllib_request,
     compat_urlparse,
     compat_xml_parse_error,
+    compat_zip as zip,
 )
 from ..downloader.f4m import (
     get_base_url,
@@ -70,6 +73,7 @@ from ..utils import (
     str_or_none,
     str_to_int,
     strip_or_none,
+    traverse_obj,
     try_get,
     unescapeHTML,
     unified_strdate,
@@ -1349,6 +1353,44 @@ class InfoExtractor(object):
                     break
         return dict((k, v) for k, v in info.items() if v is not None)
 
+    def _search_nextjs_data(self, webpage, video_id, **kw):
+        nkw = dict((k, v) for k, v in kw.items() if k in ('transform_source', 'fatal'))
+        kw.pop('transform_source', None)
+        next_data = self._search_regex(
+            r'''<script[^>]+\bid\s*=\s*('|")__NEXT_DATA__\1[^>]*>(?P<nd>[^<]+)</script>''',
+            webpage, 'next.js data', group='nd', **kw)
+        if not next_data:
+            return {}
+        return self._parse_json(next_data, video_id, **nkw)
+
+    def _search_nuxt_data(self, webpage, video_id, *args, **kwargs):
+        """Parses Nuxt.js metadata. This works as long as the function __NUXT__ invokes is a pure function"""
+
+        # self, webpage, video_id, context_name='__NUXT__', *, fatal=True, traverse=('data', 0)
+        context_name = args[0] if len(args) > 0 else kwargs.get('context_name', '__NUXT__')
+        fatal = kwargs.get('fatal', True)
+        traverse = kwargs.get('traverse', ('data', 0))
+
+        re_ctx = re.escape(context_name)
+
+        FUNCTION_RE = (r'\(\s*function\s*\((?P<arg_keys>[\s\S]*?)\)\s*\{\s*'
+                       r'return\s+(?P<js>\{[\s\S]*?})\s*;?\s*}\s*\((?P<arg_vals>[\s\S]*?)\)')
+
+        js, arg_keys, arg_vals = self._search_regex(
+            (p.format(re_ctx, FUNCTION_RE) for p in
+             (r'<script>\s*window\s*\.\s*{0}\s*=\s*{1}\s*\)\s*;?\s*</script>',
+              r'{0}\s*\([\s\S]*?{1}')),
+            webpage, context_name, group=('js', 'arg_keys', 'arg_vals'),
+            default=NO_DEFAULT if fatal else (None, None, None))
+        if js is None:
+            return {}
+
+        args = dict(zip(arg_keys.split(','), map(json.dumps, self._parse_json(
+            '[{0}]'.format(arg_vals), video_id, transform_source=js_to_json, fatal=fatal) or ())))
+
+        ret = self._parse_json(js, video_id, transform_source=functools.partial(js_to_json, vars=args), fatal=fatal)
+        return traverse_obj(ret, traverse) or {}
+
     @staticmethod
     def _hidden_inputs(html):
         html = re.sub(r'<!--(?:(?!<!--).)*-->', '', html)
@@ -2496,7 +2538,8 @@ class InfoExtractor(object):
                 return f
             return {}
 
-        def _media_formats(src, cur_media_type, type_info={}):
+        def _media_formats(src, cur_media_type, type_info=None):
+            type_info = type_info or {}
             full_url = absolute_url(src)
             ext = type_info.get('ext') or determine_ext(full_url)
             if ext == 'm3u8':
@@ -2514,6 +2557,7 @@ class InfoExtractor(object):
                 formats = [{
                     'url': full_url,
                     'vcodec': 'none' if cur_media_type == 'audio' else None,
+                    'ext': ext,
                 }]
             return is_plain_url, formats
 
@@ -2522,7 +2566,7 @@ class InfoExtractor(object):
         # so we wll include them right here (see
         # https://www.ampproject.org/docs/reference/components/amp-video)
         # For dl8-* tags see https://delight-vr.com/documentation/dl8-video/
-        _MEDIA_TAG_NAME_RE = r'(?:(?:amp|dl8(?:-live)?)-)?(video|audio)'
+        _MEDIA_TAG_NAME_RE = r'(?:(?:amp|dl8(?:-live)?)-)?(video(?:-js)?|audio)'
         media_tags = [(media_tag, media_tag_name, media_type, '')
                       for media_tag, media_tag_name, media_type
                       in re.findall(r'(?s)(<(%s)[^>]*/>)' % _MEDIA_TAG_NAME_RE, webpage)]
@@ -2540,7 +2584,8 @@ class InfoExtractor(object):
             media_attributes = extract_attributes(media_tag)
             src = strip_or_none(media_attributes.get('src'))
             if src:
-                _, formats = _media_formats(src, media_type)
+                f = parse_content_type(media_attributes.get('type'))
+                _, formats = _media_formats(src, media_type, f)
                 media_info['formats'].extend(formats)
             media_info['thumbnail'] = absolute_url(media_attributes.get('poster'))
             if media_content:
diff --git a/youtube_dl/extractor/globalplayer.py b/youtube_dl/extractor/globalplayer.py
index cceab9e6a..db490b141 100644
--- a/youtube_dl/extractor/globalplayer.py
+++ b/youtube_dl/extractor/globalplayer.py
@@ -24,13 +24,6 @@ class GlobalPlayerBaseIE(InfoExtractor):
     def _match_valid_url(cls, url):
         return cls.re.match(cls._VALID_URL, url)
 
-    def _search_nextjs_data(self, webpage, video_id, **kw):
-        return self._parse_json(
-            self._search_regex(
-                r'(?s)<script[^>]+id=[\'"]__NEXT_DATA__[\'"][^>]*>([^<]+)</script>',
-                webpage, 'next.js data', **kw),
-            video_id, **kw)
-
     def _get_page_props(self, url, video_id):
         webpage = self._download_webpage(url, video_id)
         return self._search_nextjs_data(webpage, video_id)['props']['pageProps']
@@ -39,13 +32,14 @@ class GlobalPlayerBaseIE(InfoExtractor):
         return urlhandle_detect_ext(self._request_webpage(  # Server rejects HEAD requests
             url, video_id, note='Determining source extension'))
 
-    def _extract_audio(self, episode, series):
+    @staticmethod
+    def _clean_desc(x):
+        x = clean_html(x)
+        if x:
+            x = x.replace('\xa0', ' ')
+        return x
 
-        def clean_desc(x):
-            x = clean_html(x)
-            if x:
-                x = x.replace('\xa0', ' ')
-            return x
+    def _extract_audio(self, episode, series):
 
         return merge_dicts({
             'vcodec': 'none',
@@ -56,7 +50,7 @@ class GlobalPlayerBaseIE(InfoExtractor):
             'uploader': 'itunesAuthor',  # podcasts only
         }), traverse_obj(episode, {
             'id': 'id',
-            'description': ('description', T(clean_desc)),
+            'description': ('description', T(self._clean_desc)),
             'duration': ('duration', T(parse_duration)),
             'thumbnail': 'imageUrl',
             'url': 'streamUrl',
@@ -141,9 +135,9 @@ class GlobalPlayerLivePlaylistIE(GlobalPlayerBaseIE):
             'ext': 'aac',
             # 'live_status': 'is_live',
             'is_live': True,
-            'description': 'md5:e10f5e10b01a7f2c14ba815509fbb38d',
+            'description': r're:(?s).+\bclassical\b.+\bClassic FM Hall [oO]f Fame\b',
             'thumbnail': 'https://images.globalplayer.com/images/551379?width=450&signature=oMLPZIoi5_dBSHnTMREW0Xg76mA=',
-            'title': 're:^Classic FM Hall of Fame.+$'
+            'title': 're:Classic FM Hall of Fame.+$'
         },
     }]
 
@@ -160,7 +154,7 @@ class GlobalPlayerLivePlaylistIE(GlobalPlayerBaseIE):
             'is_live': True,
         }, traverse_obj(station, {
             'title': 'title',
-            'description': 'description',
+            'description': ('description', T(self._clean_desc)),
             'thumbnail': 'image',
         }), rev=True)
 
@@ -177,7 +171,7 @@ class GlobalPlayerAudioIE(GlobalPlayerBaseIE):
             'thumbnail': 'md5:60286e7d12d795bd1bbc9efc6cee643e',
             'categories': ['Society & Culture', 'True Crime'],
             'uploader': 'Global',
-            'description': 'md5:da5b918eac9ae319454a10a563afacf9',
+            'description': r're:(?s).+\bscam\b.+?\bseries available now\b',
         },
     }, {
         # radio catchup
@@ -203,7 +197,7 @@ class GlobalPlayerAudioIE(GlobalPlayerBaseIE):
                         series, ('episodes', lambda _, v: v['id'] and v['streamUrl']))],
             'categories': traverse_obj(series, ('categories', Ellipsis, 'name')) or None,
         }, traverse_obj(series, {
-            'description': 'description',
+            'description': ('description', T(self._clean_desc)),
             'thumbnail': 'imageUrl',
             'title': 'title',
             'uploader': 'itunesAuthor',  # podcasts only
diff --git a/youtube_dl/extractor/whyp.py b/youtube_dl/extractor/whyp.py
index 16f9154ad..644eb4617 100644
--- a/youtube_dl/extractor/whyp.py
+++ b/youtube_dl/extractor/whyp.py
@@ -21,7 +21,7 @@ class WhypIE(InfoExtractor):
             'url': 'https://cdn.whyp.it/50eb17cc-e9ff-4e18-b89b-dc9206a95cb1.mp3',
             'id': '18337',
             'title': 'Home Page Example Track',
-            'description': 'md5:bd758000fb93f3159339c852b5b9133c',
+            'description': r're:(?s).+\bexample track\b',
             'ext': 'mp3',
             'duration': 52.82,
             'uploader': 'Brad',
@@ -33,29 +33,6 @@ class WhypIE(InfoExtractor):
         'only_matching': True,
     }]
 
-    def _search_nuxt_data(self, webpage, video_id, context_name='__NUXT__', fatal=True, traverse=('data', 0)):
-        """Parses Nuxt.js metadata. This works as long as the function __NUXT__ invokes is a pure function"""
-
-        import functools
-        import json
-        import re
-        from ..utils import (js_to_json, NO_DEFAULT)
-
-        re_ctx = re.escape(context_name)
-        FUNCTION_RE = r'\(function\((?P<arg_keys>.*?)\){return\s+(?P<js>{.*?})\s*;?\s*}\((?P<arg_vals>.*?)\)'
-        js, arg_keys, arg_vals = self._search_regex(
-            (p.format(re_ctx, FUNCTION_RE) for p in (r'<script>\s*window\.{0}={1}\s*\)\s*;?\s*</script>', r'{0}\(.*?{1}')),
-            webpage, context_name, group=('js', 'arg_keys', 'arg_vals'),
-            default=NO_DEFAULT if fatal else (None, None, None))
-        if js is None:
-            return {}
-
-        args = dict(zip(arg_keys.split(','), map(json.dumps, self._parse_json(
-            '[{0}]'.format(arg_vals), video_id, transform_source=js_to_json, fatal=fatal) or ())))
-
-        ret = self._parse_json(js, video_id, transform_source=functools.partial(js_to_json, vars=args), fatal=fatal)
-        return traverse_obj(ret, traverse) or {}
-
     def _real_extract(self, url):
         unique_id = self._match_id(url)
         webpage = self._download_webpage(url, unique_id)

From a190b559640ce1b5fe67e5a4843dc58328503f3c Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 19 Jul 2023 13:01:02 +0100
Subject: [PATCH 682/860] [utils] Fix broken Py 3.11+ compat in
 `traverse_obj()`

* inspect.getargspec is missing despite doc claiming backward compat
* replace with emulation of `Signature.bind()`
---
 youtube_dl/utils.py | 36 ++++++++++++++++++++++++++++++++----
 1 file changed, 32 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 0cbbec0f3..d52fa7a28 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -6109,6 +6109,37 @@ def clean_podcast_url(url):
         )/''', '', url)
 
 
+if __debug__:
+    # Raise TypeError if args can't be bound
+    # needs compat owing to unstable inspect API, thanks PSF :-(
+    try:
+        inspect.signature
+
+        def _try_bind_args(fn, *args, **kwargs):
+            inspect.signature(fn).bind(*args, **kwargs)
+    except AttributeError:
+        # Py < 3.3
+        def _try_bind_args(fn, *args, **kwargs):
+            fn_args = inspect.getargspec(fn)
+            # Py2: ArgInfo(args, varargs, keywords, defaults)
+            # Py3: ArgSpec(args, varargs, keywords, defaults)
+            if not fn_args.keywords:
+                for k in kwargs:
+                    if k not in (fn_args.args or []):
+                        raise TypeError("got an unexpected keyword argument: '{0}'".format(k))
+            if not fn_args.varargs:
+                args_to_bind = len(args)
+                bindable = len(fn_args.args or [])
+                if args_to_bind > bindable:
+                    raise TypeError('too many positional arguments')
+                bindable -= len(fn_args.defaults or [])
+                if args_to_bind < bindable:
+                    if kwargs:
+                        bindable -= len(set(fn_args.args or []) & set(kwargs))
+                    if bindable > args_to_bind:
+                        raise TypeError("missing a required argument: '{0}'".format(fn_args.args[args_to_bind]))
+
+
 def traverse_obj(obj, *paths, **kwargs):
     """
     Safely traverse nested `dict`s and `Iterable`s
@@ -6327,10 +6358,7 @@ def traverse_obj(obj, *paths, **kwargs):
 
             if __debug__ and callable(key):
                 # Verify function signature
-                args = inspect.getargspec(key)
-                if len(args.args) != 2:
-                    # crash differently in 2.6 !
-                    inspect.getcallargs(key, None, None)
+                _try_bind_args(key, None, None)
 
             new_objs = []
             for obj in objs:

From b2ba24bb026904f3503db71f65d2b1627f08edf1 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 19 Jul 2023 14:14:50 +0100
Subject: [PATCH 683/860] [InfoExtractor] Add `_match_valid_url()` class method
 and refactor

* API compatible with yt-dlp
* also support Sequence of patterns in _VALID_URL
* one place to compile _VALID_URL
* TODO: remove existing extractor shims
---
 devscripts/make_lazy_extractors.py   | 14 ++++++--
 youtube_dl/extractor/common.py       | 51 +++++++++++++++++++++-------
 youtube_dl/extractor/globalplayer.py |  6 ----
 3 files changed, 49 insertions(+), 22 deletions(-)

diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py
index a8b6ff1b9..1a841a08b 100644
--- a/devscripts/make_lazy_extractors.py
+++ b/devscripts/make_lazy_extractors.py
@@ -4,6 +4,7 @@ from inspect import getsource
 import io
 import os
 from os.path import dirname as dirn
+import re
 import sys
 
 print('WARNING: Lazy loading extractors is an experimental feature that may not always work', file=sys.stderr)
@@ -29,11 +30,18 @@ from youtube_dl.extractor.common import InfoExtractor, SearchInfoExtractor
 with open('devscripts/lazy_load_template.py', 'rt') as f:
     module_template = f.read()
 
+
+def get_source(m):
+    return re.sub(r'(?m)^\s*#.*\n', '', getsource(m))
+
+
 module_contents = [
-    module_template + '\n' + getsource(InfoExtractor.suitable) + '\n',
+    module_template,
+    get_source(InfoExtractor.suitable),
+    get_source(InfoExtractor._match_valid_url) + '\n',
     'class LazyLoadSearchExtractor(LazyLoadExtractor):\n    pass\n',
     # needed for suitable() methods of Youtube extractor (see #28780)
-    'from youtube_dl.utils import parse_qs\n',
+    'from youtube_dl.utils import parse_qs, variadic\n',
 ]
 
 ie_template = '''
@@ -66,7 +74,7 @@ def build_lazy_ie(ie, name):
         valid_url=valid_url,
         module=ie.__module__)
     if ie.suitable.__func__ is not InfoExtractor.suitable.__func__:
-        s += '\n' + getsource(ie.suitable)
+        s += '\n' + get_source(ie.suitable)
     if hasattr(ie, '_make_valid_url'):
         # search extractors
         s += make_valid_template.format(valid_url=ie._make_valid_url())
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 549781186..7f416d312 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -83,6 +83,7 @@ from ..utils import (
     urljoin,
     url_basename,
     url_or_none,
+    variadic,
     xpath_element,
     xpath_text,
     xpath_with_ns,
@@ -371,9 +372,22 @@ class InfoExtractor(object):
     title, description etc.
 
 
-    Subclasses of this one should re-define the _real_initialize() and
-    _real_extract() methods and define a _VALID_URL regexp.
-    Probably, they should also be added to the list of extractors.
+    A subclass of InfoExtractor must be defined to handle each specific site (or
+    several sites). Such a concrete subclass should be added to the list of
+    extractors. It should also:
+    * define its _VALID_URL attribute as a regexp, or a Sequence of alternative
+      regexps (but see below)
+    * re-define the _real_extract() method
+    * optionally re-define the _real_initialize() method.
+
+    An extractor subclass may also override suitable() if necessary, but the
+    function signature must be preserved and the function must import everything
+    it needs (except other extractors), so that lazy_extractors works correctly.
+    If the subclass's suitable() and _real_extract() functions avoid using
+    _VALID_URL, the subclass need not set that class attribute.
+
+    An abstract subclass of InfoExtractor may be used to simplify implementation
+    within an extractor module; it should not be added to the list of extractors.
 
     _GEO_BYPASS attribute may be set to False in order to disable
     geo restriction bypass mechanisms for a particular extractor.
@@ -408,22 +422,33 @@ class InfoExtractor(object):
         self._x_forwarded_for_ip = None
         self.set_downloader(downloader)
 
+    @classmethod
+    def __match_valid_url(cls, url):
+        # This does not use has/getattr intentionally - we want to know whether
+        # we have cached the regexp for cls, whereas getattr would also
+        # match its superclass
+        if '_VALID_URL_RE' not in cls.__dict__:
+            # _VALID_URL can now be a list/tuple of patterns
+            cls._VALID_URL_RE = tuple(map(re.compile, variadic(cls._VALID_URL)))
+        # 20% faster than next(filter(None, (p.match(url) for p in cls._VALID_URL_RE)), None) in 2.7
+        for p in cls._VALID_URL_RE:
+            p = p.match(url)
+            if p:
+                return p
+
+    # The public alias can safely be overridden, as in some back-ports
+    _match_valid_url = __match_valid_url
+
     @classmethod
     def suitable(cls, url):
         """Receives a URL and returns True if suitable for this IE."""
-
-        # This does not use has/getattr intentionally - we want to know whether
-        # we have cached the regexp for *this* class, whereas getattr would also
-        # match the superclass
-        if '_VALID_URL_RE' not in cls.__dict__:
-            cls._VALID_URL_RE = re.compile(cls._VALID_URL)
-        return cls._VALID_URL_RE.match(url) is not None
+        # This function must import everything it needs (except other extractors),
+        # so that lazy_extractors works correctly
+        return cls.__match_valid_url(url) is not None
 
     @classmethod
     def _match_id(cls, url):
-        if '_VALID_URL_RE' not in cls.__dict__:
-            cls._VALID_URL_RE = re.compile(cls._VALID_URL)
-        m = cls._VALID_URL_RE.match(url)
+        m = cls.__match_valid_url(url)
         assert m
         return compat_str(m.group('id'))
 
diff --git a/youtube_dl/extractor/globalplayer.py b/youtube_dl/extractor/globalplayer.py
index db490b141..ae75dcabf 100644
--- a/youtube_dl/extractor/globalplayer.py
+++ b/youtube_dl/extractor/globalplayer.py
@@ -18,12 +18,6 @@ from ..utils import (
 
 class GlobalPlayerBaseIE(InfoExtractor):
 
-    import re
-
-    @classmethod
-    def _match_valid_url(cls, url):
-        return cls.re.match(cls._VALID_URL, url)
-
     def _get_page_props(self, url, video_id):
         webpage = self._download_webpage(url, video_id)
         return self._search_nextjs_data(webpage, video_id)['props']['pageProps']

From 1fa8b86f0b95f2e1488042ceeda8f356ea2a5448 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 20 Jul 2023 05:29:59 +0100
Subject: [PATCH 684/860] [utils] Remove stray undocumented Host header in
 redirect (fix 46fde7c)

---
 youtube_dl/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index d52fa7a28..6d798f13a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2996,7 +2996,8 @@ class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
         # Technically the Cookie header should be in unredirected_hdrs;
         # however in practice some may set it in normal headers anyway.
         # We will remove it here to prevent any leaks.
-        remove_headers = ['Cookie']
+        # Also remove unwanted and undocumented Host header for old URL
+        remove_headers = ['Cookie', 'Host']
 
         # A 303 must either use GET or HEAD for subsequent request
         # https://datatracker.ietf.org/doc/html/rfc7231#section-6.4.4

From 74eef6bb5e6b88d042aa13caec667aa3df84ba73 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 20 Jul 2023 12:42:46 +0100
Subject: [PATCH 685/860] [workflows/ci.yml] Extend Python versions * add 3.10
 - 3.12 * use https://pypi.org/project/pynose/ for Py >= 3.9 * test Windows
 with 3.4 * set defaults (main, both) except push: (all, core)

---
 .github/workflows/ci.yml | 48 +++++++++++++++++++++++++---------------
 1 file changed, 30 insertions(+), 18 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index c3aabde47..10951d322 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -1,22 +1,34 @@
 name: CI
 
 env:
-  # add 3.10+ after patching nose (https://github.com/nose-devs/nose/issues/1099)
-  # or switching to fork of https://github.com/mdmintz/pynose
-  all-cpython-versions: 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9
-  main-cpython-versions: 2.7, 3.2, 3.5, 3.9
+  all-cpython-versions: 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12
+  main-cpython-versions: 2.7, 3.2, 3.5, 3.9, 3.11
   pypy-versions: pypy-2.7, pypy-3.6, pypy-3.7
   cpython-versions: main
-  test-set: both
+  test-set: core
 
 on:
   push:
+    inputs:
+      cpython-versions:
+        type: string
+        default: all
+      test-set:
+        type: string
+        default: core
   pull_request:
+    inputs:
+      cpython-versions:
+        type: string
+        default: main
+      test-set:
+        type: string
+        default: both
   workflow_dispatch:
     inputs:
       cpython-versions:
         type: choice
-        description: CPython versions (main = 2.7, 3.2, 3.5, 3.9)
+        description: CPython versions (main = 2.7, 3.2, 3.5, 3.9, 3.11)
         options:
           - all
           - main
@@ -30,7 +42,7 @@ on:
           - core
           - download
         required: true
-        default: core
+        default: both
 
 permissions:
   contents: read
@@ -44,7 +56,8 @@ jobs:
       test-set: ${{ steps.run.outputs.test-set }}
       own-pip-versions: ${{ steps.run.outputs.own-pip-versions }}
     steps:
-    - id: run
+    - name: Make version array
+      id: run
       run: |
         # Make a JSON Array from comma/space-separated string (no extra escaping)
         json_list() { \
@@ -66,7 +79,6 @@ jobs:
         # versions with a special get-pip.py in a per-version subdirectory
         printf 'own-pip-versions=%s\n' \
           "$(json_list 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6)" >> "$GITHUB_OUTPUT"
-
   tests:
     name: Run tests
     needs: select
@@ -82,19 +94,18 @@ jobs:
       fail-fast: true
       matrix:
         os: [ubuntu-20.04]
-        # outside steps, use github.env...., not env....
         python-version: ${{ fromJSON(needs.select.outputs.cpython-versions) }}
         python-impl: [cpython]
         ytdl-test-set: ${{ fromJSON(needs.select.outputs.test-set) }}
         run-tests-ext: [sh]
         include:
         - os: windows-2019
-          python-version: 3.2
+          python-version: 3.4
           python-impl: cpython
           ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }}
           run-tests-ext: bat
         - os: windows-2019
-          python-version: 3.2
+          python-version: 3.4
           python-impl: cpython
           ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download'  || 'nodownload' }}
           run-tests-ext: bat
@@ -205,17 +216,14 @@ jobs:
           make install )
         rm -rf $openssl_name
         rmdir $openssl_ssl/certs && ln -s /etc/ssl/certs $openssl_ssl/certs
-
         # Download PyEnv from its GitHub repository.
         export PYENV_ROOT=${{ env.PYENV_ROOT }}
         export PATH=$PYENV_ROOT/bin:$PATH
         git clone "https://github.com/pyenv/pyenv.git" "$PYENV_ROOT"
-
         # Prevent pyenv build trying (and failing) to update pip
         export GET_PIP=get-pip-2.6.py
         echo 'import sys; sys.exit(0)' > ${GET_PIP}
         GET_PIP=$(realpath $GET_PIP)
-
         # Build and install Python
         export CFLAGS="-I$openssl_inc"
         export LDFLAGS="-L$openssl_lib"
@@ -322,7 +330,12 @@ jobs:
       run: |
         echo "$PATH"
         echo "$PYTHONHOME"
-        $PIP -qq show nose || $PIP install nose
+        # Use PyNose for recent Pythons instead of Nose
+        py3ver="${{ matrix.python-version }}"
+        py3ver=${py3ver#3.}
+        [ "$py3ver" != "${{ matrix.python-version }}" ] && py3ver=${py3ver%.*} || py3ver=0
+        [ "$py3ver" -ge 9 ] && nose=pynose || nose=nose
+        $PIP -qq show $nose || $PIP install $nose
     - name: Install nose for other Python 2
       if: ${{ matrix.python-impl == 'jython' || matrix.python-version == '2.6' }}
       shell: bash
@@ -354,7 +367,7 @@ jobs:
           '    def setUp(self):' \
           '        self.ver = os.environ["PYTHON_VER"].split("-")' \
           '    def test_python_ver(self):' \
-          '        self.assertEqual(sys.version[:3], self.ver[-1])' \
+          '        self.assertEqual(["%d" % v for v in sys.version_info[:2]], self.ver[-1].split(".")[:2])' \
           '        self.assertTrue(sys.version.startswith(self.ver[-1]))' \
           '        self.assertIn(self.ver[0], sys.version.lower())' \
           '    def test_python_impl(self):' \
@@ -370,7 +383,6 @@ jobs:
         PYTHON_IMPL: ${{ matrix.python-impl }}
       run: |
         ./devscripts/run_tests.${{ matrix.run-tests-ext }}
-
   flake8:
     name: Linter
     runs-on: ubuntu-latest

From 2a4e9faa773cce60e82453cb32f13e48513c4a46 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 20 Jul 2023 15:49:46 +0100
Subject: [PATCH 686/860] [doc] Update developer guidance * mention pynose *
 mention traverse_obj and add/revise examples

[skip ci]
---
 README.md | 113 ++++++++++++++++++++++++++++++++++++++++++++++++------
 1 file changed, 101 insertions(+), 12 deletions(-)

diff --git a/README.md b/README.md
index 14a3d6c86..47e686f84 100644
--- a/README.md
+++ b/README.md
@@ -33,7 +33,7 @@ Windows users can [download an .exe file](https://yt-dl.org/latest/youtube-dl.ex
 You can also use pip:
 
     sudo -H pip install --upgrade youtube-dl
-    
+
 This command will update youtube-dl if you have already installed it. See the [pypi page](https://pypi.python.org/pypi/youtube_dl) for more information.
 
 macOS users can install youtube-dl with [Homebrew](https://brew.sh/):
@@ -563,7 +563,7 @@ The basic usage is not to set any template arguments when downloading a single f
  - `is_live` (boolean): Whether this video is a live stream or a fixed-length video
  - `start_time` (numeric): Time in seconds where the reproduction should start, as specified in the URL
  - `end_time` (numeric): Time in seconds where the reproduction should end, as specified in the URL
- - `format` (string): A human-readable description of the format 
+ - `format` (string): A human-readable description of the format
  - `format_id` (string): Format code specified by `--format`
  - `format_note` (string): Additional info about the format
  - `width` (numeric): Width of the video
@@ -675,7 +675,7 @@ The general syntax for format selection is `--format FORMAT` or shorter `-f FORM
 
 **tl;dr:** [navigate me to examples](#format-selection-examples).
 
-The simplest case is requesting a specific format, for example with `-f 22` you can download the format with format code equal to 22. You can get the list of available format codes for particular video using `--list-formats` or `-F`. Note that these format codes are extractor specific. 
+The simplest case is requesting a specific format, for example with `-f 22` you can download the format with format code equal to 22. You can get the list of available format codes for particular video using `--list-formats` or `-F`. Note that these format codes are extractor specific.
 
 You can also use a file extension (currently `3gp`, `aac`, `flv`, `m4a`, `mp3`, `mp4`, `ogg`, `wav`, `webm` are supported) to download the best quality format of a particular file extension served as a single file, e.g. `-f webm` will download the best quality format with the `webm` extension served as a single file.
 
@@ -760,7 +760,7 @@ Videos can be filtered by their upload date using the options `--date`, `--dateb
 
  - Absolute dates: Dates in the format `YYYYMMDD`.
  - Relative dates: Dates in the format `(now|today)[+-][0-9](day|week|month|year)(s)?`
- 
+
 Examples:
 
 ```bash
@@ -1000,6 +1000,8 @@ To run the test, simply invoke your favorite test runner, or execute a test file
     python test/test_download.py
     nosetests
 
+For Python versions 3.6 and later, you can use [pynose](https://pypi.org/project/pynose/) to implement `nosetests`. The original [nose](https://pypi.org/project/nose/) has not been upgraded for 3.10 and later.
+
 See item 6 of [new extractor tutorial](#adding-support-for-a-new-site) for how to run extractor specific test cases.
 
 If you want to create a build of youtube-dl yourself, you'll need
@@ -1091,7 +1093,7 @@ In any case, thank you very much for your contributions!
 
 ## youtube-dl coding conventions
 
-This section introduces a guide lines for writing idiomatic, robust and future-proof extractor code.
+This section introduces guidelines for writing idiomatic, robust and future-proof extractor code.
 
 Extractors are very fragile by nature since they depend on the layout of the source data provided by 3rd party media hosters out of your control and this layout tends to change. As an extractor implementer your task is not only to write code that will extract media links and metadata correctly but also to minimize dependency on the source's layout and even to make the code foresee potential future changes and be ready for that. This is important because it will allow the extractor not to break on minor layout changes thus keeping old youtube-dl versions working. Even though this breakage issue is easily fixed by emitting a new version of youtube-dl with a fix incorporated, all the previous versions become broken in all repositories and distros' packages that may not be so prompt in fetching the update from us. Needless to say, some non rolling release distros may never receive an update at all.
 
@@ -1114,7 +1116,7 @@ Say you have some source dictionary `meta` that you've fetched as JSON with HTTP
 ```python
 meta = self._download_json(url, video_id)
 ```
-    
+
 Assume at this point `meta`'s layout is:
 
 ```python
@@ -1158,7 +1160,7 @@ description = self._search_regex(
 ```
 
 On failure this code will silently continue the extraction with `description` set to `None`. That is useful for metafields that may or may not be present.
- 
+
 ### Provide fallbacks
 
 When extracting metadata try to do so from multiple sources. For example if `title` is present in several places, try extracting from at least some of them. This makes it more future-proof in case some of the sources become unavailable.
@@ -1206,7 +1208,7 @@ r'(id|ID)=(?P<id>\d+)'
 #### Make regular expressions relaxed and flexible
 
 When using regular expressions try to write them fuzzy, relaxed and flexible, skipping insignificant parts that are more likely to change, allowing both single and double quotes for quoted values and so on.
- 
+
 ##### Example
 
 Say you need to extract `title` from the following HTML code:
@@ -1230,7 +1232,7 @@ title = self._search_regex(
     webpage, 'title', group='title')
 ```
 
-Note how you tolerate potential changes in the `style` attribute's value or switch from using double quotes to single for `class` attribute: 
+Note how you tolerate potential changes in the `style` attribute's value or switch from using double quotes to single for `class` attribute:
 
 The code definitely should not look like:
 
@@ -1331,27 +1333,114 @@ Wrap all extracted numeric data into safe functions from [`youtube_dl/utils.py`]
 
 Use `url_or_none` for safe URL processing.
 
-Use `try_get` for safe metadata extraction from parsed JSON.
+Use `traverse_obj` for safe metadata extraction from parsed JSON.
 
-Use `unified_strdate` for uniform `upload_date` or any `YYYYMMDD` meta field extraction, `unified_timestamp` for uniform `timestamp` extraction, `parse_filesize` for `filesize` extraction, `parse_count` for count meta fields extraction, `parse_resolution`, `parse_duration` for `duration` extraction, `parse_age_limit` for `age_limit` extraction. 
+Use `unified_strdate` for uniform `upload_date` or any `YYYYMMDD` meta field extraction, `unified_timestamp` for uniform `timestamp` extraction, `parse_filesize` for `filesize` extraction, `parse_count` for count meta fields extraction, `parse_resolution`, `parse_duration` for `duration` extraction, `parse_age_limit` for `age_limit` extraction.
 
 Explore [`youtube_dl/utils.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/utils.py) for more useful convenience functions.
 
 #### More examples
 
 ##### Safely extract optional description from parsed JSON
+
+When processing complex JSON, as often returned by site API requests or stashed in web pages for "hydration", you can use the `traverse_obj()` utility function to handle multiple fallback values and to ensure the expected type of metadata items. The function's docstring defines how the function works: also review usage in the codebase for more examples.
+
+In this example, a text `description`, or `None`, is pulled from the `.result.video[0].summary` member of the parsed JSON `response`, if available.
+
+```python
+description = traverse_obj(response, ('result', 'video', 0, 'summary', T(compat_str)))
+```
+`T(...)` is a shorthand for a set literal; if you hate people who still run Python 2.6, `T(type_or_transformation)` could be written as a set literal `{type_or_transformation}`.
+
+Some extractors use the older and less capable `try_get()` function in the same way.
+
 ```python
 description = try_get(response, lambda x: x['result']['video'][0]['summary'], compat_str)
 ```
 
 ##### Safely extract more optional metadata
+
+In this example, various optional metadata values are extracted from the `.result.video[0]` member of the parsed JSON `response`, which is expected to be a JS object, parsed into a `dict`, with no crash if that isn't so, or if any of the target values are missing or invalid.
+
 ```python
-video = try_get(response, lambda x: x['result']['video'][0], dict) or {}
+video = traverse_obj(response, ('result', 'video', 0, T(dict))) or {}
+# formerly:
+# video = try_get(response, lambda x: x['result']['video'][0], dict) or {}
 description = video.get('summary')
 duration = float_or_none(video.get('durationMs'), scale=1000)
 view_count = int_or_none(video.get('views'))
 ```
 
+#### Safely extract nested lists
+
+Suppose you've extracted JSON like this into a Python data structure named `media_json` using, say, the `_download_json()` or `_parse_json()` methods of `InfoExtractor`:
+```json
+{
+    "title": "Example video",
+    "comment": "try extracting this",
+    "media": [{
+        "type": "bad",
+        "size": 320,
+        "url": "https://some.cdn.site/bad.mp4"
+    }, {
+        "type": "streaming",
+        "url": "https://some.cdn.site/hls.m3u8"
+    }, {
+        "type": "super",
+        "size": 1280,
+        "url": "https://some.cdn.site/good.webm"
+    }],
+    "moreStuff": "more values",
+    ...
+}
+```
+
+Then extractor code like this can collect the various fields of the JSON:
+```python
+...
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    T,
+    traverse_obj,
+    txt_or_none,
+    url_or_none,
+)
+...
+        ...
+        info_dict = {}
+        # extract title and description if valid and not empty
+        info_dict.update(traverse_obj(media_json, {
+            'title': ('title', T(txt_or_none)),
+            'description': ('comment', T(txt_or_none)),
+        }))
+
+        # extract any recognisable media formats
+        fmts = []
+        # traverse into "media" list, extract `dict`s with desired keys
+        for fmt in traverse_obj(media_json, ('media', Ellipsis, {
+                'format_id': ('type', T(txt_or_none)),
+                'url': ('url', T(url_or_none)),
+                'width': ('size', T(int_or_none)), })):
+            # bad `fmt` values were `None` and removed
+            if 'url' not in fmt:
+                continue
+            fmt_url = fmt['url']  # known to be valid URL
+            ext = determine_ext(fmt_url)
+            if ext == 'm3u8':
+                fmts.extend(self._extract_m3u8_formats(fmt_url, video_id, 'mp4', fatal=False))
+            else:
+                fmt['ext'] = ext
+                fmts.append(fmt)
+
+        # sort, raise if no formats
+        self._sort_formats(fmts)
+
+        info_dict['formats'] = fmts
+        ...
+```
+The extractor raises an exception rather than random crashes if the JSON structure changes so that no formats are found.
+
 # EMBEDDING YOUTUBE-DL
 
 youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/ytdl-org/youtube-dl/issues/new).

From ca71e56c481c6d5ce69b4756f8f8c0aff97d79b5 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 20 Jul 2023 16:36:54 +0100
Subject: [PATCH 687/860] [workflows/ci.yml] Build 3.12 with pyenv

---
 .github/workflows/ci.yml | 40 ++++++++++++++++++++++++++++++++++++++--
 1 file changed, 38 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 10951d322..a1e21fd4a 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -124,7 +124,7 @@ jobs:
     #-------- Python 3 -----
     - name: Set up supported Python ${{ matrix.python-version }}
       id: setup-python
-      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version != '2.6' && matrix.python-version != '2.7'}}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version != '2.6' && matrix.python-version != '2.7' && matrix.python-version != '3.12'}}
       # wrap broken actions/setup-python@v4
       uses: ytdl-org/setup-python@v1
       with:
@@ -162,6 +162,42 @@ jobs:
             'import sys' \
             'print(sys.path)' \
             | ${expected} -
+    #-------- Python 3.12 -
+    - name: Set up Python 3.12 environment
+      if: ${{ matrix.python-version == '3.12' }}
+      shell: bash
+      run: |
+        PYENV_ROOT=$HOME/.local/share/pyenv
+        echo "PYENV_ROOT=${PYENV_ROOT}" >> "$GITHUB_ENV"
+    - name: Cache Python 3.12
+      id: cache312
+      if: ${{ matrix.python-version == '3.12' }}
+      uses: actions/cache@v3
+      with:
+        key: python-3.12
+        path: |
+          ${{ env.PYENV_ROOT }}
+    - name: Build and set up Python 3.12
+      if: ${{ matrix.python-version == '3.12' && ! steps.cache312.outputs.cache-hit }}
+      # dl and build locally
+      shell: bash
+      run: |
+        # Install build environment
+        sudo apt-get install -y build-essential llvm libssl-dev tk-dev  \
+                      libncursesw5-dev libreadline-dev libsqlite3-dev   \
+                      libffi-dev xz-utils zlib1g-dev libbz2-dev liblzma-dev
+        # Download PyEnv from its GitHub repository.
+        export PYENV_ROOT=${{ env.PYENV_ROOT }}
+        export PATH=$PYENV_ROOT/bin:$PATH
+        git clone "https://github.com/pyenv/pyenv.git" "$PYENV_ROOT"
+        pyenv install 3.12.0b4
+    - name: Locate Python 3.12
+      if: ${{ matrix.python-version == '3.12' }}
+      shell: bash
+      run: |
+        PYTHONHOME="${{ env.PYENV_ROOT }}/versions/3.12.0b4"
+        echo "PYTHONHOME=$PYTHONHOME" >> "$GITHUB_ENV"
+        echo "PATH=${PYTHONHOME}/bin:$PATH" >> "$GITHUB_ENV"
     #-------- Python 2.7 --
     - name: Set up Python 2.7
       if: ${{ matrix.python-version == '2.7' }}
@@ -325,7 +361,7 @@ jobs:
         done
     #-------- nose --------
     - name: Install nose for Python ${{ matrix.python-version }}
-      if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.7' }}
+      if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.7' || matrix.python-version == '3.12' }}
       shell: bash
       run: |
         echo "$PATH"

From 7bce2ad441b874e7a1cf8cc81059c5601d832697 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 20 Jul 2023 18:49:48 +0100
Subject: [PATCH 688/860] [build] Fix various Jython CI and test issues

---
 .github/workflows/ci.yml           | 38 +++++++++++++++++++-----------
 devscripts/make_lazy_extractors.py | 11 +++++++++
 test/test_http.py                  | 13 ++++++----
 youtube_dl/compat.py               |  2 +-
 4 files changed, 45 insertions(+), 19 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a1e21fd4a..6b91edd6c 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -111,10 +111,12 @@ jobs:
           run-tests-ext: bat
         # jython
         - os: ubuntu-20.04
+          python-version: 2.7
           python-impl: jython
           ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }}
           run-tests-ext: sh
         - os: ubuntu-20.04
+          python-version: 2.7
           python-impl: jython
           ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download'  || 'nodownload' }}
           run-tests-ext: sh
@@ -163,22 +165,22 @@ jobs:
             'print(sys.path)' \
             | ${expected} -
     #-------- Python 3.12 -
-    - name: Set up Python 3.12 environment
-      if: ${{ matrix.python-version == '3.12' }}
+    - name: Set up CPython 3.12 environment
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }}
       shell: bash
       run: |
         PYENV_ROOT=$HOME/.local/share/pyenv
         echo "PYENV_ROOT=${PYENV_ROOT}" >> "$GITHUB_ENV"
     - name: Cache Python 3.12
       id: cache312
-      if: ${{ matrix.python-version == '3.12' }}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }}
       uses: actions/cache@v3
       with:
         key: python-3.12
         path: |
           ${{ env.PYENV_ROOT }}
     - name: Build and set up Python 3.12
-      if: ${{ matrix.python-version == '3.12' && ! steps.cache312.outputs.cache-hit }}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' && ! steps.cache312.outputs.cache-hit }}
       # dl and build locally
       shell: bash
       run: |
@@ -192,7 +194,7 @@ jobs:
         git clone "https://github.com/pyenv/pyenv.git" "$PYENV_ROOT"
         pyenv install 3.12.0b4
     - name: Locate Python 3.12
-      if: ${{ matrix.python-version == '3.12' }}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }}
       shell: bash
       run: |
         PYTHONHOME="${{ env.PYENV_ROOT }}/versions/3.12.0b4"
@@ -200,7 +202,7 @@ jobs:
         echo "PATH=${PYTHONHOME}/bin:$PATH" >> "$GITHUB_ENV"
     #-------- Python 2.7 --
     - name: Set up Python 2.7
-      if: ${{ matrix.python-version == '2.7' }}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.7' }}
       # install 2.7
       shell: bash
       run: |
@@ -208,7 +210,7 @@ jobs:
         echo "PYTHONHOME=/usr" >> "$GITHUB_ENV"
     #-------- Python 2.6 --
     - name: Set up Python 2.6 environment
-      if: ${{ matrix.python-version == '2.6' }}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' }}
       shell: bash
       run: |
         openssl_name=openssl-1.0.2u
@@ -228,7 +230,7 @@ jobs:
           ${{ env.openssl_dir }}
           ${{ env.PYENV_ROOT }}
     - name: Build and set up Python 2.6
-      if: ${{ matrix.python-version == '2.6' && ! steps.cache26.outputs.cache-hit }}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' && ! steps.cache26.outputs.cache-hit }}
       # dl and build locally
       shell: bash
       run: |
@@ -266,7 +268,7 @@ jobs:
         export LD_LIBRARY_PATH="$openssl_lib"
         pyenv install 2.6.9
     - name: Locate Python 2.6
-      if: ${{ matrix.python-version == '2.6' }}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' }}
       shell: bash
       run: |
         PYTHONHOME="${{ env.PYENV_ROOT }}/versions/2.6.9"
@@ -288,7 +290,7 @@ jobs:
         echo "PIP=pip" >> "$GITHUB_ENV"
     - name: Cache Jython
       id: cachejy
-      if: ${{ matrix.python-impl == 'jython' }}
+      if: ${{ matrix.python-impl == 'jython' && matrix.python-version == '2.7' }}
       uses: actions/cache@v3
       with:
         # 2.7.3 now available, may solve SNI issue
@@ -296,7 +298,7 @@ jobs:
         path: |
           ${{ env.JYTHON_ROOT }}
     - name: Install Jython
-      if: ${{ matrix.python-impl == 'jython' && ! steps.cachejy.outputs.cache-hit }}
+      if: ${{ matrix.python-impl == 'jython' && matrix.python-version == '2.7' && ! steps.cachejy.outputs.cache-hit }}
       shell: bash
       run: |
         JYTHON_ROOT="${{ env.JYTHON_ROOT }}"
@@ -309,6 +311,11 @@ jobs:
       run: |
         JYTHON_ROOT="${{ env.JYTHON_ROOT }}"
         echo "${JYTHON_ROOT}/bin" >> $GITHUB_PATH
+    - name: Install supporting Python 2.7 if possible
+      if: ${{ steps.cachejy.outputs.cache-hit }}
+      shell: bash
+      run: |
+        sudo apt-get install -y python2.7 || true
     #-------- pip ---------
     - name: Set up supported Python ${{ matrix.python-version }} pip
       if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.7' }}
@@ -391,6 +398,11 @@ jobs:
       if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }}
       shell: bash
       run: |
+        # set PYTHON_VER
+        PYTHON_VER=${{ matrix.python-version }}
+        [ "${PYTHON_VER#*-}" != "$PYTHON_VER" ] || PYTHON_VER="${{ matrix.python-impl }}-${PYTHON_VER}"
+        echo "PYTHON_VER=$PYTHON_VER" >> "$GITHUB_ENV"
+        echo "PYTHON_IMPL=${{ matrix.python-impl }}" >> "$GITHUB_ENV"
         # define a test to validate the Python version used by nosetests
         printf '%s\n' \
           'from __future__ import unicode_literals' \
@@ -405,7 +417,7 @@ jobs:
           '    def test_python_ver(self):' \
           '        self.assertEqual(["%d" % v for v in sys.version_info[:2]], self.ver[-1].split(".")[:2])' \
           '        self.assertTrue(sys.version.startswith(self.ver[-1]))' \
-          '        self.assertIn(self.ver[0], sys.version.lower())' \
+          '        self.assertIn(self.ver[0], ",".join((sys.version, platform.python_implementation())).lower())' \
           '    def test_python_impl(self):' \
           '        self.assertIn(platform.python_implementation().lower(), (os.environ["PYTHON_IMPL"], self.ver[0]))' \
           > test/test_python.py
@@ -415,8 +427,6 @@ jobs:
       continue-on-error: ${{ matrix.ytdl-test-set == 'download' || matrix.python-impl == 'jython' }}
       env:
         YTDL_TEST_SET: ${{ matrix.ytdl-test-set }}
-        PYTHON_VER: ${{ matrix.python-version }}
-        PYTHON_IMPL: ${{ matrix.python-impl }}
       run: |
         ./devscripts/run_tests.${{ matrix.run-tests-ext }}
   flake8:
diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py
index 1a841a08b..dee9d6d91 100644
--- a/devscripts/make_lazy_extractors.py
+++ b/devscripts/make_lazy_extractors.py
@@ -118,3 +118,14 @@ module_src = '\n'.join(module_contents) + '\n'
 
 with io.open(lazy_extractors_filename, 'wt', encoding='utf-8') as f:
     f.write(module_src)
+
+# work around JVM byte code module limit in Jython
+if sys.platform.startswith('java') and sys.version_info[:2] == (2, 7):
+    import subprocess
+    from youtube_dl.compat import compat_subprocess_get_DEVNULL
+    # if Python 2.7 is available, use it to compile the module for Jython
+    try:
+        # if Python 2.7 is available, use it to compile the module for Jython
+        subprocess.check_call(['python2.7', '-m', 'py_compile', lazy_extractors_filename], stdout=compat_subprocess_get_DEVNULL())
+    except Exception:
+        pass
diff --git a/test/test_http.py b/test/test_http.py
index 1a6b2e878..4ec8e13e3 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -45,6 +45,7 @@ from youtube_dl.utils import (
 )
 
 from test.helper import (
+    expectedFailureIf,
     FakeYDL,
     FakeLogger,
     http_server_port,
@@ -243,6 +244,11 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
 
 
 class TestHTTP(unittest.TestCase):
+    # when does it make sense to check the SSL certificate?
+    _check_cert = (
+        sys.version_info >= (3, 2)
+        or (sys.version_info[0] == 2 and sys.version_info[1:] >= (7, 19)))
+
     def setUp(self):
         # HTTP server
         self.http_httpd = compat_http_server.HTTPServer(
@@ -307,10 +313,7 @@ class TestHTTP(unittest.TestCase):
             else self.https_port if scheme == 'https'
             else self.http_port, path)
 
-    @unittest.skipUnless(
-        sys.version_info >= (3, 2)
-        or (sys.version_info[0] == 2 and sys.version_info[1:] >= (7, 9)),
-        'No support for certificate check in SSL')
+    @unittest.skipUnless(_check_cert, 'No support for certificate check in SSL')
     def test_nocheckcertificate(self):
         with FakeYDL({'logger': FakeLogger()}) as ydl:
             with self.assertRaises(compat_urllib_error.URLError):
@@ -376,6 +379,8 @@ class TestHTTP(unittest.TestCase):
                 with self.assertRaises(compat_urllib_HTTPError):
                     do_req(code, 'GET')
 
+    # Jython 2.7.1 times out for some reason
+    @expectedFailureIf(sys.platform.startswith('java') and sys.version_info < (2, 7, 2))
     def test_content_type(self):
         # https://github.com/yt-dlp/yt-dlp/commit/379a4f161d4ad3e40932dcf5aca6e6fb9715ab28
         with FakeYDL({'nocheckcertificate': True}) as ydl:
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 1d784d90f..da6d70ec4 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -131,7 +131,7 @@ if sys.version_info[0] == 2 or sys.version_info < (3, 3):
         def load(self, rawdata):
             must_have_value = 0
             if not isinstance(rawdata, dict):
-                if sys.version_info[:2] != (2, 7):
+                if sys.version_info[:2] != (2, 7) or sys.platform.startswith('java'):
                     # attribute must have value for parsing
                     rawdata, must_have_value = re.subn(
                         r'(?i)(;\s*)(secure|httponly)(\s*(?:;|$))', r'\1\2=\2\3', rawdata)

From 44faa71b19c866b836e4433ddd3e4722ac6d282f Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 20 Jul 2023 19:32:29 +0100
Subject: [PATCH 689/860] [test/test_execution.py] Use
 `compat_subprocess_get_DEVNULL()`

---
 test/test_execution.py | 7 ++-----
 1 file changed, 2 insertions(+), 5 deletions(-)

diff --git a/test/test_execution.py b/test/test_execution.py
index ae59e562a..56e1b679d 100644
--- a/test/test_execution.py
+++ b/test/test_execution.py
@@ -10,17 +10,14 @@ import os
 import subprocess
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-from youtube_dl.compat import compat_register_utf8
+from youtube_dl.compat import compat_register_utf8, compat_subprocess_get_DEVNULL
 from youtube_dl.utils import encodeArgument
 
 compat_register_utf8()
 
 rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
-try:
-    _DEV_NULL = subprocess.DEVNULL
-except AttributeError:
-    _DEV_NULL = open(os.devnull, 'wb')
+_DEV_NULL = compat_subprocess_get_DEVNULL()
 
 
 class TestExecution(unittest.TestCase):

From 2b7dd3b2a2d7c6e228a42d1000a6f3296739ff1c Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 24 Jul 2023 03:30:28 +0100
Subject: [PATCH 690/860] [utils] Fix update_Request() with empty data (not
 None)

---
 test/test_http.py   | 13 +++++++++++++
 youtube_dl/utils.py |  7 +++----
 2 files changed, 16 insertions(+), 4 deletions(-)

diff --git a/test/test_http.py b/test/test_http.py
index 4ec8e13e3..89580969d 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -41,6 +41,7 @@ from youtube_dl.compat import (
 
 from youtube_dl.utils import (
     sanitized_Request,
+    update_Request,
     urlencode_postdata,
 )
 
@@ -395,6 +396,18 @@ class TestHTTP(unittest.TestCase):
             headers = ydl.urlopen(r).read().decode('utf-8')
             self.assertIn('Content-Type: application/x-www-form-urlencoded', headers)
 
+    def test_update_req(self):
+        req = sanitized_Request('http://example.com')
+        assert req.data is None
+        assert req.get_method() == 'GET'
+        assert not req.has_header('Content-Type')
+        # Test that zero-byte payloads will be sent
+        req = update_Request(req, data=b'')
+        assert req.data == b''
+        assert req.get_method() == 'POST'
+        # yt-dl expects data to be encoded and Content-Type to be added by sender
+        # assert req.get_header('Content-Type') == 'application/x-www-form-urlencoded'
+
     def test_cookiejar(self):
         with FakeYDL() as ydl:
             ydl.cookiejar.set_cookie(compat_http_cookiejar_Cookie(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 6d798f13a..b5475434f 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2996,8 +2996,7 @@ class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
         # Technically the Cookie header should be in unredirected_hdrs;
         # however in practice some may set it in normal headers anyway.
         # We will remove it here to prevent any leaks.
-        # Also remove unwanted and undocumented Host header for old URL
-        remove_headers = ['Cookie', 'Host']
+        remove_headers = ['Cookie']
 
         # A 303 must either use GET or HEAD for subsequent request
         # https://datatracker.ietf.org/doc/html/rfc7231#section-6.4.4
@@ -3016,7 +3015,7 @@ class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
             remove_headers.extend(['Content-Length', 'Content-Type'])
 
         # NB: don't use dict comprehension for python 2.6 compatibility
-        new_headers = dict((k, v) for k, v in req.header_items()
+        new_headers = dict((k, v) for k, v in req.headers.items()
                            if k.title() not in remove_headers)
 
         return compat_urllib_request.Request(
@@ -4187,7 +4186,7 @@ def update_url_query(url, query):
 def update_Request(req, url=None, data=None, headers={}, query={}):
     req_headers = req.headers.copy()
     req_headers.update(headers)
-    req_data = data or req.data
+    req_data = data if data is not None else req.data
     req_url = update_url_query(url or req.get_full_url(), query)
     req_get_method = req.get_method()
     if req_get_method == 'HEAD':

From aac33155e40af3da96a2467dd05faea201815989 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 24 Jul 2023 23:43:36 +0100
Subject: [PATCH 691/860] [build] Add and use `devscripts/utils`

---
 devscripts/__init__.py             |  1 +
 devscripts/make_lazy_extractors.py | 22 +++++------
 devscripts/utils.py                | 62 ++++++++++++++++++++++++++++++
 test/test_execution.py             | 10 +++--
 4 files changed, 80 insertions(+), 15 deletions(-)
 create mode 100644 devscripts/__init__.py
 create mode 100644 devscripts/utils.py

diff --git a/devscripts/__init__.py b/devscripts/__init__.py
new file mode 100644
index 000000000..750dbdca7
--- /dev/null
+++ b/devscripts/__init__.py
@@ -0,0 +1 @@
+# Empty file needed to make devscripts.utils properly importable from outside
diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py
index dee9d6d91..5b8b123a4 100644
--- a/devscripts/make_lazy_extractors.py
+++ b/devscripts/make_lazy_extractors.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals, print_function
 
 from inspect import getsource
-import io
 import os
 from os.path import dirname as dirn
 import re
@@ -9,17 +8,20 @@ import sys
 
 print('WARNING: Lazy loading extractors is an experimental feature that may not always work', file=sys.stderr)
 
-sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
+sys.path.insert(0, dirn(dirn(os.path.abspath(__file__))))
 
 lazy_extractors_filename = sys.argv[1]
 if os.path.exists(lazy_extractors_filename):
     os.remove(lazy_extractors_filename)
 # Py2: may be confused by leftover lazy_extractors.pyc
-try:
-    os.remove(lazy_extractors_filename + 'c')
-except OSError:
-    pass
+if sys.version_info[0] < 3:
+    for c in ('c', 'o'):
+        try:
+            os.remove(lazy_extractors_filename + 'c')
+        except OSError:
+            pass
 
+from devscripts.utils import read_file, write_file
 from youtube_dl.compat import compat_register_utf8
 
 compat_register_utf8()
@@ -27,8 +29,7 @@ compat_register_utf8()
 from youtube_dl.extractor import _ALL_CLASSES
 from youtube_dl.extractor.common import InfoExtractor, SearchInfoExtractor
 
-with open('devscripts/lazy_load_template.py', 'rt') as f:
-    module_template = f.read()
+module_template = read_file('devscripts/lazy_load_template.py')
 
 
 def get_source(m):
@@ -114,10 +115,9 @@ for ie in ordered_cls:
 module_contents.append(
     '_ALL_CLASSES = [{0}]'.format(', '.join(names)))
 
-module_src = '\n'.join(module_contents) + '\n'
+module_src = '\n'.join(module_contents)
 
-with io.open(lazy_extractors_filename, 'wt', encoding='utf-8') as f:
-    f.write(module_src)
+write_file(lazy_extractors_filename, module_src + '\n')
 
 # work around JVM byte code module limit in Jython
 if sys.platform.startswith('java') and sys.version_info[:2] == (2, 7):
diff --git a/devscripts/utils.py b/devscripts/utils.py
new file mode 100644
index 000000000..2d072d2e0
--- /dev/null
+++ b/devscripts/utils.py
@@ -0,0 +1,62 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+import argparse
+import functools
+import os.path
+import subprocess
+import sys
+
+dirn = os.path.dirname
+
+sys.path.insert(0, dirn(dirn(os.path.abspath(__file__))))
+
+from youtube_dl.compat import (
+    compat_kwargs,
+    compat_open as open,
+)
+
+
+def read_file(fname):
+    with open(fname, encoding='utf-8') as f:
+        return f.read()
+
+
+def write_file(fname, content, mode='w'):
+    with open(fname, mode, encoding='utf-8') as f:
+        return f.write(content)
+
+
+def read_version(fname='youtube_dl/version.py'):
+    """Get the version without importing the package"""
+    exec(compile(read_file(fname), fname, 'exec'))
+    return locals()['__version__']
+
+
+def get_filename_args(has_infile=False, default_outfile=None):
+    parser = argparse.ArgumentParser()
+    if has_infile:
+        parser.add_argument('infile', help='Input file')
+    kwargs = {'nargs': '?', 'default': default_outfile} if default_outfile else {}
+    kwargs['help'] = 'Output file'
+    parser.add_argument('outfile', **compat_kwargs(kwargs))
+
+    opts = parser.parse_args()
+    if has_infile:
+        return opts.infile, opts.outfile
+    return opts.outfile
+
+
+def compose_functions(*functions):
+    return lambda x: functools.reduce(lambda y, f: f(y), functions, x)
+
+
+def run_process(*args, **kwargs):
+    kwargs.setdefault('text', True)
+    kwargs.setdefault('check', True)
+    kwargs.setdefault('capture_output', True)
+    if kwargs['text']:
+        kwargs.setdefault('encoding', 'utf-8')
+        kwargs.setdefault('errors', 'replace')
+        kwargs = compat_kwargs(kwargs)
+    return subprocess.run(args, **kwargs)
diff --git a/test/test_execution.py b/test/test_execution.py
index 56e1b679d..9daaafa6c 100644
--- a/test/test_execution.py
+++ b/test/test_execution.py
@@ -8,14 +8,16 @@ import unittest
 import sys
 import os
 import subprocess
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+
+sys.path.insert(0, rootDir)
 
 from youtube_dl.compat import compat_register_utf8, compat_subprocess_get_DEVNULL
 from youtube_dl.utils import encodeArgument
 
 compat_register_utf8()
 
-rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 
 _DEV_NULL = compat_subprocess_get_DEVNULL()
 
@@ -49,10 +51,10 @@ class TestExecution(unittest.TestCase):
             subprocess.check_call([sys.executable, os.path.normpath('devscripts/make_lazy_extractors.py'), lazy_extractors], cwd=rootDir, stdout=_DEV_NULL)
             subprocess.check_call([sys.executable, os.path.normpath('test/test_all_urls.py')], cwd=rootDir, stdout=_DEV_NULL)
         finally:
-            for x in ['', 'c'] if sys.version_info[0] < 3 else ['']:
+            for x in ('', 'c') if sys.version_info[0] < 3 else ('',):
                 try:
                     os.remove(lazy_extractors + x)
-                except (IOError, OSError):
+                except OSError:
                     pass
 
 

From a25e9f3c84a34d43f78a4e5a6f6c2e98e2a0ade3 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 25 Jul 2023 00:17:15 +0100
Subject: [PATCH 692/860] [compat] Use `compat_open()`

---
 devscripts/make_readme.py                  |  4 +++-
 test/helper.py                             |  6 +++---
 test/test_InfoExtractor.py                 | 18 +++++++++---------
 test/test_YoutubeDL.py                     |  7 ++++---
 test/test_download.py                      |  6 +++---
 test/test_swfinterp.py                     | 10 ++++++----
 test/test_unicode_literals.py              | 12 +++++++-----
 test/test_write_annotations.py             |  5 ++---
 test/test_youtube_signature.py             |  9 ++++++---
 youtube_dl/YoutubeDL.py                    | 19 +++++++------------
 youtube_dl/cache.py                        |  8 +++++---
 youtube_dl/extractor/common.py             |  1 +
 youtube_dl/extractor/openload.py           |  1 +
 youtube_dl/postprocessor/embedthumbnail.py |  2 ++
 youtube_dl/postprocessor/ffmpeg.py         |  8 ++++----
 youtube_dl/update.py                       |  7 +++++--
 16 files changed, 68 insertions(+), 55 deletions(-)

diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py
index 8fbce0796..c5d5dd4f1 100755
--- a/devscripts/make_readme.py
+++ b/devscripts/make_readme.py
@@ -4,6 +4,8 @@ import io
 import sys
 import re
 
+from youtube_dl.compat import compat_open as open
+
 README_FILE = 'README.md'
 helptext = sys.stdin.read()
 
@@ -20,7 +22,7 @@ options = helptext[helptext.index('  General Options:') + 19:]
 options = re.sub(r'(?m)^  (\w.+)$', r'## \1', options)
 options = '# OPTIONS\n' + options + '\n'
 
-with io.open(README_FILE, 'w', encoding='utf-8') as f:
+with open(README_FILE, 'w', encoding='utf-8') as f:
     f.write(header)
     f.write(options)
     f.write(footer)
diff --git a/test/helper.py b/test/helper.py
index aa99001b2..fc55c6b46 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -1,7 +1,6 @@
 from __future__ import unicode_literals
 
 import errno
-import io
 import hashlib
 import json
 import os.path
@@ -14,6 +13,7 @@ import unittest
 import youtube_dl.extractor
 from youtube_dl import YoutubeDL
 from youtube_dl.compat import (
+    compat_open as open,
     compat_os_name,
     compat_str,
 )
@@ -29,10 +29,10 @@ def get_params(override=None):
                                    "parameters.json")
     LOCAL_PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)),
                                          "local_parameters.json")
-    with io.open(PARAMETERS_FILE, encoding='utf-8') as pf:
+    with open(PARAMETERS_FILE, encoding='utf-8') as pf:
         parameters = json.load(pf)
     if os.path.exists(LOCAL_PARAMETERS_FILE):
-        with io.open(LOCAL_PARAMETERS_FILE, encoding='utf-8') as pf:
+        with open(LOCAL_PARAMETERS_FILE, encoding='utf-8') as pf:
             parameters.update(json.load(pf))
     if override:
         parameters.update(override)
diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 34773fbd0..3f96645de 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -3,7 +3,6 @@
 from __future__ import unicode_literals
 
 # Allow direct execution
-import io
 import os
 import sys
 import unittest
@@ -21,6 +20,7 @@ from test.helper import (
 from youtube_dl.compat import (
     compat_etree_fromstring,
     compat_http_server,
+    compat_open as open,
 )
 from youtube_dl.extractor.common import InfoExtractor
 from youtube_dl.extractor import (
@@ -902,8 +902,8 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
         ]
 
         for m3u8_file, m3u8_url, expected_formats in _TEST_CASES:
-            with io.open('./test/testdata/m3u8/%s.m3u8' % m3u8_file,
-                         mode='r', encoding='utf-8') as f:
+            with open('./test/testdata/m3u8/%s.m3u8' % m3u8_file,
+                      mode='r', encoding='utf-8') as f:
                 formats = self.ie._parse_m3u8_formats(
                     f.read(), m3u8_url, ext='mp4')
                 self.ie._sort_formats(formats)
@@ -1127,8 +1127,8 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
         ]
 
         for mpd_file, mpd_url, mpd_base_url, expected_formats in _TEST_CASES:
-            with io.open('./test/testdata/mpd/%s.mpd' % mpd_file,
-                         mode='r', encoding='utf-8') as f:
+            with open('./test/testdata/mpd/%s.mpd' % mpd_file,
+                      mode='r', encoding='utf-8') as f:
                 formats = self.ie._parse_mpd_formats(
                     compat_etree_fromstring(f.read().encode('utf-8')),
                     mpd_base_url=mpd_base_url, mpd_url=mpd_url)
@@ -1154,8 +1154,8 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
         ]
 
         for f4m_file, f4m_url, expected_formats in _TEST_CASES:
-            with io.open('./test/testdata/f4m/%s.f4m' % f4m_file,
-                         mode='r', encoding='utf-8') as f:
+            with open('./test/testdata/f4m/%s.f4m' % f4m_file,
+                      mode='r', encoding='utf-8') as f:
                 formats = self.ie._parse_f4m_formats(
                     compat_etree_fromstring(f.read().encode('utf-8')),
                     f4m_url, None)
@@ -1202,8 +1202,8 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
         ]
 
         for xspf_file, xspf_url, expected_entries in _TEST_CASES:
-            with io.open('./test/testdata/xspf/%s.xspf' % xspf_file,
-                         mode='r', encoding='utf-8') as f:
+            with open('./test/testdata/xspf/%s.xspf' % xspf_file,
+                      mode='r', encoding='utf-8') as f:
                 entries = self.ie._parse_xspf(
                     compat_etree_fromstring(f.read().encode('utf-8')),
                     xspf_file, xspf_url=xspf_url, xspf_base_url=xspf_url)
diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py
index 6cf555827..d994682b2 100644
--- a/test/test_YoutubeDL.py
+++ b/test/test_YoutubeDL.py
@@ -22,6 +22,7 @@ from youtube_dl.compat import (
     compat_http_cookiejar_Cookie,
     compat_http_cookies_SimpleCookie,
     compat_kwargs,
+    compat_open as open,
     compat_str,
     compat_urllib_error,
 )
@@ -701,12 +702,12 @@ class TestYoutubeDL(unittest.TestCase):
 
         class SimplePP(PostProcessor):
             def run(self, info):
-                with open(audiofile, 'wt') as f:
+                with open(audiofile, 'w') as f:
                     f.write('EXAMPLE')
                 return [info['filepath']], info
 
         def run_pp(params, PP):
-            with open(filename, 'wt') as f:
+            with open(filename, 'w') as f:
                 f.write('EXAMPLE')
             ydl = YoutubeDL(params)
             ydl.add_post_processor(PP())
@@ -725,7 +726,7 @@ class TestYoutubeDL(unittest.TestCase):
 
         class ModifierPP(PostProcessor):
             def run(self, info):
-                with open(info['filepath'], 'wt') as f:
+                with open(info['filepath'], 'w') as f:
                     f.write('MODIFIED')
                 return [], info
 
diff --git a/test/test_download.py b/test/test_download.py
index d50008307..e0bc8cb95 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -20,15 +20,15 @@ from test.helper import (
 
 
 import hashlib
-import io
 import json
 import socket
 
 import youtube_dl.YoutubeDL
 from youtube_dl.compat import (
     compat_http_client,
-    compat_urllib_error,
     compat_HTTPError,
+    compat_open as open,
+    compat_urllib_error,
 )
 from youtube_dl.utils import (
     DownloadError,
@@ -245,7 +245,7 @@ def generator(test_case, tname):
                 self.assertTrue(
                     os.path.exists(info_json_fn),
                     'Missing info file %s' % info_json_fn)
-                with io.open(info_json_fn, encoding='utf-8') as infof:
+                with open(info_json_fn, encoding='utf-8') as infof:
                     info_dict = json.load(infof)
                 expect_info_dict(self, info_dict, tc.get('info_dict', {}))
         finally:
diff --git a/test/test_swfinterp.py b/test/test_swfinterp.py
index 9f18055e6..7c282ee00 100644
--- a/test/test_swfinterp.py
+++ b/test/test_swfinterp.py
@@ -5,16 +5,18 @@ from __future__ import unicode_literals
 import os
 import sys
 import unittest
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
+dirn = os.path.dirname
+
+sys.path.insert(0, dirn(dirn(os.path.abspath(__file__))))
 
 import errno
-import io
 import json
 import re
 import subprocess
 
 from youtube_dl.swfinterp import SWFInterpreter
+from youtube_dl.compat import compat_open as open
 
 
 TEST_DIR = os.path.join(
@@ -43,7 +45,7 @@ def _make_testfunc(testfile):
                     '-static-link-runtime-shared-libraries', as_file])
             except OSError as ose:
                 if ose.errno == errno.ENOENT:
-                    print('mxmlc not found! Skipping test.')
+                    self.skipTest('mxmlc not found!')
                     return
                 raise
 
@@ -51,7 +53,7 @@ def _make_testfunc(testfile):
             swf_content = swf_f.read()
         swfi = SWFInterpreter(swf_content)
 
-        with io.open(as_file, 'r', encoding='utf-8') as as_f:
+        with open(as_file, 'r', encoding='utf-8') as as_f:
             as_content = as_f.read()
 
         def _find_spec(key):
diff --git a/test/test_unicode_literals.py b/test/test_unicode_literals.py
index c7c2252f5..0c83f2a0c 100644
--- a/test/test_unicode_literals.py
+++ b/test/test_unicode_literals.py
@@ -2,14 +2,15 @@ from __future__ import unicode_literals
 
 # Allow direct execution
 import os
+import re
 import sys
 import unittest
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-import io
-import re
+dirn = os.path.dirname
 
-rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+rootDir = dirn(dirn(os.path.abspath(__file__)))
+
+sys.path.insert(0, rootDir)
 
 IGNORED_FILES = [
     'setup.py',  # http://bugs.python.org/issue13943
@@ -24,6 +25,7 @@ IGNORED_DIRS = [
 ]
 
 from test.helper import assertRegexpMatches
+from youtube_dl.compat import compat_open as open
 
 
 class TestUnicodeLiterals(unittest.TestCase):
@@ -41,7 +43,7 @@ class TestUnicodeLiterals(unittest.TestCase):
                     continue
 
                 fn = os.path.join(dirpath, basename)
-                with io.open(fn, encoding='utf-8') as inf:
+                with open(fn, encoding='utf-8') as inf:
                     code = inf.read()
 
                 if "'" not in code and '"' not in code:
diff --git a/test/test_write_annotations.py b/test/test_write_annotations.py
index 41abdfe3b..68e0a391d 100644
--- a/test/test_write_annotations.py
+++ b/test/test_write_annotations.py
@@ -11,12 +11,11 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 from test.helper import get_params, try_rm
 
 
-import io
-
 import xml.etree.ElementTree
 
 import youtube_dl.YoutubeDL
 import youtube_dl.extractor
+from youtube_dl.compat import compat_open as open
 
 
 class YoutubeDL(youtube_dl.YoutubeDL):
@@ -51,7 +50,7 @@ class TestAnnotations(unittest.TestCase):
         ydl.download([TEST_ID])
         self.assertTrue(os.path.exists(ANNOTATIONS_FILE))
         annoxml = None
-        with io.open(ANNOTATIONS_FILE, 'r', encoding='utf-8') as annof:
+        with open(ANNOTATIONS_FILE, 'r', encoding='utf-8') as annof:
             annoxml = xml.etree.ElementTree.parse(annof)
         self.assertTrue(annoxml is not None, 'Failed to parse annotations XML')
         root = annoxml.getroot()
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 5dcabaf95..f45dfec7c 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -8,11 +8,14 @@ import sys
 import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
-import io
 import re
 import string
 
-from youtube_dl.compat import compat_str, compat_urlretrieve
+from youtube_dl.compat import (
+    compat_open as open,
+    compat_str,
+    compat_urlretrieve,
+)
 
 from test.helper import FakeYDL
 from youtube_dl.extractor import YoutubeIE
@@ -208,7 +211,7 @@ def t_factory(name, sig_func, url_pattern):
 
             if not os.path.exists(fn):
                 compat_urlretrieve(url, fn)
-            with io.open(fn, encoding='utf-8') as testf:
+            with open(fn, encoding='utf-8') as testf:
                 jscode = testf.read()
             self.assertEqual(sig_func(jscode, sig_input), expected_sig)
 
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 98d080f43..6a12f91e4 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -4,11 +4,9 @@
 from __future__ import absolute_import, unicode_literals
 
 import collections
-import contextlib
 import copy
 import datetime
 import errno
-import fileinput
 import io
 import itertools
 import json
@@ -45,6 +43,7 @@ from .compat import (
     compat_kwargs,
     compat_map as map,
     compat_numeric_types,
+    compat_open as open,
     compat_os_name,
     compat_str,
     compat_tokenize_tokenize,
@@ -1977,7 +1976,7 @@ class YoutubeDL(object):
             else:
                 try:
                     self.to_screen('[info] Writing video description to: ' + descfn)
-                    with io.open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
+                    with open(encodeFilename(descfn), 'w', encoding='utf-8') as descfile:
                         descfile.write(info_dict['description'])
                 except (OSError, IOError):
                     self.report_error('Cannot write description file ' + descfn)
@@ -1992,7 +1991,7 @@ class YoutubeDL(object):
             else:
                 try:
                     self.to_screen('[info] Writing video annotations to: ' + annofn)
-                    with io.open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
+                    with open(encodeFilename(annofn), 'w', encoding='utf-8') as annofile:
                         annofile.write(info_dict['annotations'])
                 except (KeyError, TypeError):
                     self.report_warning('There are no annotations to write.')
@@ -2019,7 +2018,7 @@ class YoutubeDL(object):
                         try:
                             # Use newline='' to prevent conversion of newline characters
                             # See https://github.com/ytdl-org/youtube-dl/issues/10268
-                            with io.open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
+                            with open(encodeFilename(sub_filename), 'w', encoding='utf-8', newline='') as subfile:
                                 subfile.write(sub_info['data'])
                         except (OSError, IOError):
                             self.report_error('Cannot write subtitles file ' + sub_filename)
@@ -2028,7 +2027,7 @@ class YoutubeDL(object):
                         try:
                             sub_data = ie._request_webpage(
                                 sub_info['url'], info_dict['id'], note=False).read()
-                            with io.open(encodeFilename(sub_filename), 'wb') as subfile:
+                            with open(encodeFilename(sub_filename), 'wb') as subfile:
                                 subfile.write(sub_data)
                         except (ExtractorError, IOError, OSError, ValueError) as err:
                             self.report_warning('Unable to download subtitle for "%s": %s' %
@@ -2232,12 +2231,8 @@ class YoutubeDL(object):
         return self._download_retcode
 
     def download_with_info_file(self, info_filename):
-        with contextlib.closing(fileinput.FileInput(
-                [info_filename], mode='r',
-                openhook=fileinput.hook_encoded('utf-8'))) as f:
-            # FileInput doesn't have a read method, we can't call json.load
-            # TODO: let's use io.open(), then
-            info = self.filter_requested_info(json.loads('\n'.join(f)))
+        with open(info_filename, encoding='utf-8') as f:
+            info = self.filter_requested_info(json.load(f))
         try:
             self.process_ie_result(info, download=True)
         except DownloadError:
diff --git a/youtube_dl/cache.py b/youtube_dl/cache.py
index 4822439d0..54123da0e 100644
--- a/youtube_dl/cache.py
+++ b/youtube_dl/cache.py
@@ -1,14 +1,16 @@
 from __future__ import unicode_literals
 
 import errno
-import io
 import json
 import os
 import re
 import shutil
 import traceback
 
-from .compat import compat_getenv
+from .compat import (
+    compat_getenv,
+    compat_open as open,
+)
 from .utils import (
     error_to_compat_str,
     expand_path,
@@ -83,7 +85,7 @@ class Cache(object):
         cache_fn = self._get_cache_fn(section, key, dtype)
         try:
             try:
-                with io.open(cache_fn, 'r', encoding='utf-8') as cachef:
+                with open(cache_fn, 'r', encoding='utf-8') as cachef:
                     return self._validate(json.load(cachef), min_ver)
             except ValueError:
                 try:
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 7f416d312..0eca9f844 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -25,6 +25,7 @@ from ..compat import (
     compat_integer_types,
     compat_http_client,
     compat_map as map,
+    compat_open as open,
     compat_os_name,
     compat_str,
     compat_urllib_error,
diff --git a/youtube_dl/extractor/openload.py b/youtube_dl/extractor/openload.py
index b05d60435..45b1add73 100644
--- a/youtube_dl/extractor/openload.py
+++ b/youtube_dl/extractor/openload.py
@@ -7,6 +7,7 @@ import subprocess
 import tempfile
 
 from ..compat import (
+    compat_open as open,
     compat_urlparse,
     compat_kwargs,
 )
diff --git a/youtube_dl/postprocessor/embedthumbnail.py b/youtube_dl/postprocessor/embedthumbnail.py
index 5e7b6e2df..b6c60e127 100644
--- a/youtube_dl/postprocessor/embedthumbnail.py
+++ b/youtube_dl/postprocessor/embedthumbnail.py
@@ -18,6 +18,8 @@ from ..utils import (
     shell_quote,
 )
 
+from ..compat import compat_open as open
+
 
 class EmbedThumbnailPPError(PostProcessingError):
     pass
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 8c29c8d59..801160e6c 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import io
 import os
 import subprocess
 import time
@@ -9,6 +8,7 @@ import re
 
 from .common import AudioConversionError, PostProcessor
 
+from ..compat import compat_open as open
 from ..utils import (
     encodeArgument,
     encodeFilename,
@@ -493,7 +493,7 @@ class FFmpegMetadataPP(FFmpegPostProcessor):
         chapters = info.get('chapters', [])
         if chapters:
             metadata_filename = replace_extension(filename, 'meta')
-            with io.open(metadata_filename, 'wt', encoding='utf-8') as f:
+            with open(metadata_filename, 'w', encoding='utf-8') as f:
                 def ffmpeg_escape(text):
                     return re.sub(r'(=|;|#|\\|\n)', r'\\\1', text)
 
@@ -636,7 +636,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
                 with open(dfxp_file, 'rb') as f:
                     srt_data = dfxp2srt(f.read())
 
-                with io.open(srt_file, 'wt', encoding='utf-8') as f:
+                with open(srt_file, 'w', encoding='utf-8') as f:
                     f.write(srt_data)
                 old_file = srt_file
 
@@ -652,7 +652,7 @@ class FFmpegSubtitlesConvertorPP(FFmpegPostProcessor):
 
             self.run_ffmpeg(old_file, new_file, ['-f', new_format])
 
-            with io.open(new_file, 'rt', encoding='utf-8') as f:
+            with open(new_file, 'r', encoding='utf-8') as f:
                 subs[lang] = {
                     'ext': new_ext,
                     'data': f.read(),
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index 84c964617..b5f26e4a9 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -9,7 +9,10 @@ import subprocess
 import sys
 from zipimport import zipimporter
 
-from .compat import compat_realpath
+from .compat import (
+    compat_open as open,
+    compat_realpath,
+)
 from .utils import encode_compat_str
 
 from .version import __version__
@@ -127,7 +130,7 @@ def update_self(to_screen, verbose, opener):
 
         try:
             bat = os.path.join(directory, 'youtube-dl-updater.bat')
-            with io.open(bat, 'w') as batfile:
+            with open(bat, 'w') as batfile:
                 batfile.write('''
 @echo off
 echo Waiting for file handle to be closed ...

From b87018122995acb7e6a1be3f2464605259b93611 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 25 Jul 2023 00:22:54 +0100
Subject: [PATCH 693/860] [build] Extend use of `devscripts/utils`

---
 devscripts/bash-completion.py            | 11 +++++++----
 devscripts/create-github-release.py      |  9 +++++----
 devscripts/fish-completion.py            | 11 ++++++-----
 devscripts/gh-pages/add-version.py       | 15 ++++++++++-----
 devscripts/gh-pages/generate-download.py | 17 ++++++++++++-----
 devscripts/gh-pages/update-copyright.py  | 17 +++++++++++------
 devscripts/gh-pages/update-feed.py       | 11 ++++++++---
 devscripts/gh-pages/update-sites.py      | 11 ++++++-----
 devscripts/make_contributing.py          |  9 ++++-----
 devscripts/make_issue_template.py        | 17 +++++++----------
 devscripts/make_readme.py                | 11 +++++++----
 devscripts/make_supportedsites.py        | 15 ++++++++-------
 devscripts/prepare_manpage.py            | 10 ++++------
 devscripts/zsh-completion.py             |  8 ++++----
 youtube_dl/update.py                     |  1 -
 15 files changed, 99 insertions(+), 74 deletions(-)

diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py
index 3d1391334..7db396a77 100755
--- a/devscripts/bash-completion.py
+++ b/devscripts/bash-completion.py
@@ -5,8 +5,12 @@ import os
 from os.path import dirname as dirn
 import sys
 
-sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
+sys.path.insert(0, dirn(dirn(os.path.abspath(__file__))))
+
 import youtube_dl
+from youtube_dl.compat import compat_open as open
+
+from utils import read_file
 
 BASH_COMPLETION_FILE = "youtube-dl.bash-completion"
 BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in"
@@ -18,9 +22,8 @@ def build_completion(opt_parser):
         for option in group.option_list:
             # for every long flag
             opts_flag.append(option.get_opt_string())
-    with open(BASH_COMPLETION_TEMPLATE) as f:
-        template = f.read()
-    with open(BASH_COMPLETION_FILE, "w") as f:
+    template = read_file(BASH_COMPLETION_TEMPLATE)
+    with open(BASH_COMPLETION_FILE, "w", encoding='utf-8') as f:
         # just using the special char
         filled_template = template.replace("{{flags}}", " ".join(opts_flag))
         f.write(filled_template)
diff --git a/devscripts/create-github-release.py b/devscripts/create-github-release.py
index 2ddfa1096..320bcfc27 100644
--- a/devscripts/create-github-release.py
+++ b/devscripts/create-github-release.py
@@ -1,7 +1,6 @@
 #!/usr/bin/env python
 from __future__ import unicode_literals
 
-import io
 import json
 import mimetypes
 import netrc
@@ -10,7 +9,9 @@ import os
 import re
 import sys
 
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+dirn = os.path.dirname
+
+sys.path.insert(0, dirn(dirn(os.path.abspath(__file__))))
 
 from youtube_dl.compat import (
     compat_basestring,
@@ -22,6 +23,7 @@ from youtube_dl.utils import (
     make_HTTPS_handler,
     sanitized_Request,
 )
+from utils import read_file
 
 
 class GitHubReleaser(object):
@@ -89,8 +91,7 @@ def main():
 
     changelog_file, version, build_path = args
 
-    with io.open(changelog_file, encoding='utf-8') as inf:
-        changelog = inf.read()
+    changelog = read_file(changelog_file)
 
     mobj = re.search(r'(?s)version %s\n{2}(.+?)\n{3}' % version, changelog)
     body = mobj.group(1) if mobj else ''
diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py
index 51d19dd33..267ba6a58 100755
--- a/devscripts/fish-completion.py
+++ b/devscripts/fish-completion.py
@@ -6,10 +6,13 @@ import os
 from os.path import dirname as dirn
 import sys
 
-sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
+sys.path.insert(0, dirn(dirn(os.path.abspath(__file__))))
+
 import youtube_dl
 from youtube_dl.utils import shell_quote
 
+from utils import read_file, write_file
+
 FISH_COMPLETION_FILE = 'youtube-dl.fish'
 FISH_COMPLETION_TEMPLATE = 'devscripts/fish-completion.in'
 
@@ -38,11 +41,9 @@ def build_completion(opt_parser):
             complete_cmd.extend(EXTRA_ARGS.get(long_option, []))
             commands.append(shell_quote(complete_cmd))
 
-    with open(FISH_COMPLETION_TEMPLATE) as f:
-        template = f.read()
+    template = read_file(FISH_COMPLETION_TEMPLATE)
     filled_template = template.replace('{{commands}}', '\n'.join(commands))
-    with open(FISH_COMPLETION_FILE, 'w') as f:
-        f.write(filled_template)
+    write_file(filled_template)
 
 
 parser = youtube_dl.parseOpts()[0]
diff --git a/devscripts/gh-pages/add-version.py b/devscripts/gh-pages/add-version.py
index 867ea0048..b84908f85 100755
--- a/devscripts/gh-pages/add-version.py
+++ b/devscripts/gh-pages/add-version.py
@@ -6,16 +6,21 @@ import sys
 import hashlib
 import os.path
 
+dirn = os.path.dirname
+
+sys.path.insert(0, dirn(dirn(dirn(os.path.abspath(__file__)))))
+
+from devscripts.utils import read_file, write_file
+from youtube_dl.compat import compat_open as open
 
 if len(sys.argv) <= 1:
     print('Specify the version number as parameter')
     sys.exit()
 version = sys.argv[1]
 
-with open('update/LATEST_VERSION', 'w') as f:
-    f.write(version)
+write_file('update/LATEST_VERSION', version)
 
-versions_info = json.load(open('update/versions.json'))
+versions_info = json.loads(read_file('update/versions.json'))
 if 'signature' in versions_info:
     del versions_info['signature']
 
@@ -39,5 +44,5 @@ for key, filename in filenames.items():
 versions_info['versions'][version] = new_version
 versions_info['latest'] = version
 
-with open('update/versions.json', 'w') as jsonf:
-    json.dump(versions_info, jsonf, indent=4, sort_keys=True)
+with open('update/versions.json', 'w', encoding='utf-8') as jsonf:
+    json.dumps(versions_info, jsonf, indent=4, sort_keys=True)
diff --git a/devscripts/gh-pages/generate-download.py b/devscripts/gh-pages/generate-download.py
index a873d32ee..3e38e9299 100755
--- a/devscripts/gh-pages/generate-download.py
+++ b/devscripts/gh-pages/generate-download.py
@@ -2,14 +2,21 @@
 from __future__ import unicode_literals
 
 import json
+import os.path
+import sys
 
-versions_info = json.load(open('update/versions.json'))
+dirn = os.path.dirname
+
+sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
+
+from utils import read_file, write_file
+
+versions_info = json.loads(read_file('update/versions.json'))
 version = versions_info['latest']
 version_dict = versions_info['versions'][version]
 
 # Read template page
-with open('download.html.in', 'r', encoding='utf-8') as tmplf:
-    template = tmplf.read()
+template = read_file('download.html.in')
 
 template = template.replace('@PROGRAM_VERSION@', version)
 template = template.replace('@PROGRAM_URL@', version_dict['bin'][0])
@@ -18,5 +25,5 @@ template = template.replace('@EXE_URL@', version_dict['exe'][0])
 template = template.replace('@EXE_SHA256SUM@', version_dict['exe'][1])
 template = template.replace('@TAR_URL@', version_dict['tar'][0])
 template = template.replace('@TAR_SHA256SUM@', version_dict['tar'][1])
-with open('download.html', 'w', encoding='utf-8') as dlf:
-    dlf.write(template)
+
+write_file('download.html', template)
diff --git a/devscripts/gh-pages/update-copyright.py b/devscripts/gh-pages/update-copyright.py
index 61487f925..444595c48 100755
--- a/devscripts/gh-pages/update-copyright.py
+++ b/devscripts/gh-pages/update-copyright.py
@@ -5,17 +5,22 @@ from __future__ import with_statement, unicode_literals
 
 import datetime
 import glob
-import io  # For Python 2 compatibility
 import os
 import re
+import sys
 
-year = str(datetime.datetime.now().year)
+dirn = os.path.dirname
+
+sys.path.insert(0, dirn(dirn(dirn(os.path.abspath(__file__)))))
+
+from devscripts.utils import read_file, write_file
+from youtube_dl import compat_str
+
+year = compat_str(datetime.datetime.now().year)
 for fn in glob.glob('*.html*'):
-    with io.open(fn, encoding='utf-8') as f:
-        content = f.read()
+    content = read_file(fn)
     newc = re.sub(r'(?P<copyright>Copyright © 2011-)(?P<year>[0-9]{4})', 'Copyright © 2011-' + year, content)
     if content != newc:
         tmpFn = fn + '.part'
-        with io.open(tmpFn, 'wt', encoding='utf-8') as outf:
-            outf.write(newc)
+        write_file(tmpFn, newc)
         os.rename(tmpFn, fn)
diff --git a/devscripts/gh-pages/update-feed.py b/devscripts/gh-pages/update-feed.py
index 506a62377..13a367d34 100755
--- a/devscripts/gh-pages/update-feed.py
+++ b/devscripts/gh-pages/update-feed.py
@@ -2,10 +2,16 @@
 from __future__ import unicode_literals
 
 import datetime
-import io
 import json
+import os.path
 import textwrap
+import sys
 
+dirn = os.path.dirname
+
+sys.path.insert(0, dirn(dirn(os.path.abspath(__file__))))
+
+from utils import write_file
 
 atom_template = textwrap.dedent("""\
     <?xml version="1.0" encoding="utf-8"?>
@@ -72,5 +78,4 @@ for v in versions:
 entries_str = textwrap.indent(''.join(entries), '\t')
 atom_template = atom_template.replace('@ENTRIES@', entries_str)
 
-with io.open('update/releases.atom', 'w', encoding='utf-8') as atom_file:
-    atom_file.write(atom_template)
+write_file('update/releases.atom', atom_template)
diff --git a/devscripts/gh-pages/update-sites.py b/devscripts/gh-pages/update-sites.py
index 531c93c70..06a8a474c 100755
--- a/devscripts/gh-pages/update-sites.py
+++ b/devscripts/gh-pages/update-sites.py
@@ -5,15 +5,17 @@ import sys
 import os
 import textwrap
 
+dirn = os.path.dirname
+
 # We must be able to import youtube_dl
-sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))))
+sys.path.insert(0, dirn(dirn(dirn(os.path.abspath(__file__)))))
 
 import youtube_dl
+from devscripts.utils import read_file, write_file
 
 
 def main():
-    with open('supportedsites.html.in', 'r', encoding='utf-8') as tmplf:
-        template = tmplf.read()
+    template = read_file('supportedsites.html.in')
 
     ie_htmls = []
     for ie in youtube_dl.list_extractors(age_limit=None):
@@ -29,8 +31,7 @@ def main():
 
     template = template.replace('@SITES@', textwrap.indent('\n'.join(ie_htmls), '\t'))
 
-    with open('supportedsites.html', 'w', encoding='utf-8') as sitesf:
-        sitesf.write(template)
+    write_file('supportedsites.html', template)
 
 
 if __name__ == '__main__':
diff --git a/devscripts/make_contributing.py b/devscripts/make_contributing.py
index 226d1a5d6..5a9eb194f 100755
--- a/devscripts/make_contributing.py
+++ b/devscripts/make_contributing.py
@@ -1,10 +1,11 @@
 #!/usr/bin/env python
 from __future__ import unicode_literals
 
-import io
 import optparse
 import re
 
+from utils import read_file, write_file
+
 
 def main():
     parser = optparse.OptionParser(usage='%prog INFILE OUTFILE')
@@ -14,8 +15,7 @@ def main():
 
     infile, outfile = args
 
-    with io.open(infile, encoding='utf-8') as inf:
-        readme = inf.read()
+    readme = read_file(infile)
 
     bug_text = re.search(
         r'(?s)#\s*BUGS\s*[^\n]*\s*(.*?)#\s*COPYRIGHT', readme).group(1)
@@ -25,8 +25,7 @@ def main():
 
     out = bug_text + dev_text
 
-    with io.open(outfile, 'w', encoding='utf-8') as outf:
-        outf.write(out)
+    write_file(outfile, out)
 
 
 if __name__ == '__main__':
diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py
index b7ad23d83..65fa8169f 100644
--- a/devscripts/make_issue_template.py
+++ b/devscripts/make_issue_template.py
@@ -1,8 +1,11 @@
 #!/usr/bin/env python
 from __future__ import unicode_literals
 
-import io
 import optparse
+import os.path
+import sys
+
+from utils import read_file, read_version, write_file
 
 
 def main():
@@ -13,17 +16,11 @@ def main():
 
     infile, outfile = args
 
-    with io.open(infile, encoding='utf-8') as inf:
-        issue_template_tmpl = inf.read()
+    issue_template_tmpl = read_file(infile)
 
-    # Get the version from youtube_dl/version.py without importing the package
-    exec(compile(open('youtube_dl/version.py').read(),
-                 'youtube_dl/version.py', 'exec'))
+    out = issue_template_tmpl % {'version': read_version()}
 
-    out = issue_template_tmpl % {'version': locals()['__version__']}
-
-    with io.open(outfile, 'w', encoding='utf-8') as outf:
-        outf.write(out)
+    write_file(outfile, out)
 
 if __name__ == '__main__':
     main()
diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py
index c5d5dd4f1..7a5b04dcc 100755
--- a/devscripts/make_readme.py
+++ b/devscripts/make_readme.py
@@ -1,9 +1,13 @@
 from __future__ import unicode_literals
 
-import io
-import sys
+import os.path
 import re
+import sys
+dirn = os.path.dirname
 
+sys.path.insert(0, dirn(dirn(os.path.abspath(__file__))))
+
+from utils import read_file
 from youtube_dl.compat import compat_open as open
 
 README_FILE = 'README.md'
@@ -12,8 +16,7 @@ helptext = sys.stdin.read()
 if isinstance(helptext, bytes):
     helptext = helptext.decode('utf-8')
 
-with io.open(README_FILE, encoding='utf-8') as f:
-    oldreadme = f.read()
+oldreadme = read_file(README_FILE)
 
 header = oldreadme[:oldreadme.index('# OPTIONS')]
 footer = oldreadme[oldreadme.index('# CONFIGURATION'):]
diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py
index 764795bc5..c424d18d7 100644
--- a/devscripts/make_supportedsites.py
+++ b/devscripts/make_supportedsites.py
@@ -1,17 +1,19 @@
 #!/usr/bin/env python
 from __future__ import unicode_literals
 
-import io
 import optparse
-import os
+import os.path
 import sys
 
-
 # Import youtube_dl
-ROOT_DIR = os.path.join(os.path.dirname(__file__), '..')
-sys.path.insert(0, ROOT_DIR)
+dirn = os.path.dirname
+
+sys.path.insert(0, dirn(dirn(os.path.abspath(__file__))))
+
 import youtube_dl
 
+from utils import write_file
+
 
 def main():
     parser = optparse.OptionParser(usage='%prog OUTFILE.md')
@@ -38,8 +40,7 @@ def main():
         ' - ' + md + '\n'
         for md in gen_ies_md(ies))
 
-    with io.open(outfile, 'w', encoding='utf-8') as outf:
-        outf.write(out)
+    write_file(outfile, out)
 
 
 if __name__ == '__main__':
diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py
index 76bf873e1..0090ada3e 100644
--- a/devscripts/prepare_manpage.py
+++ b/devscripts/prepare_manpage.py
@@ -1,13 +1,13 @@
 from __future__ import unicode_literals
 
-import io
 import optparse
 import os.path
 import re
 
+from utils import read_file, write_file
+
 ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
 README_FILE = os.path.join(ROOT_DIR, 'README.md')
-
 PREFIX = r'''%YOUTUBE-DL(1)
 
 # NAME
@@ -29,8 +29,7 @@ def main():
 
     outfile, = args
 
-    with io.open(README_FILE, encoding='utf-8') as f:
-        readme = f.read()
+    readme = read_file(README_FILE)
 
     readme = re.sub(r'(?s)^.*?(?=# DESCRIPTION)', '', readme)
     readme = re.sub(r'\s+youtube-dl \[OPTIONS\] URL \[URL\.\.\.\]', '', readme)
@@ -38,8 +37,7 @@ def main():
 
     readme = filter_options(readme)
 
-    with io.open(outfile, 'w', encoding='utf-8') as outf:
-        outf.write(readme)
+    write_file(outfile, readme)
 
 
 def filter_options(readme):
diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py
index 60aaf76cc..ebd552fcb 100755
--- a/devscripts/zsh-completion.py
+++ b/devscripts/zsh-completion.py
@@ -7,6 +7,8 @@ import sys
 
 sys.path.insert(0, dirn(dirn((os.path.abspath(__file__)))))
 import youtube_dl
+from utils import read_file, write_file
+
 
 ZSH_COMPLETION_FILE = "youtube-dl.zsh"
 ZSH_COMPLETION_TEMPLATE = "devscripts/zsh-completion.in"
@@ -34,15 +36,13 @@ def build_completion(opt_parser):
 
     flags = [opt.get_opt_string() for opt in opts]
 
-    with open(ZSH_COMPLETION_TEMPLATE) as f:
-        template = f.read()
+    template = read_file(ZSH_COMPLETION_TEMPLATE)
 
     template = template.replace("{{fileopts}}", "|".join(fileopts))
     template = template.replace("{{diropts}}", "|".join(diropts))
     template = template.replace("{{flags}}", " ".join(flags))
 
-    with open(ZSH_COMPLETION_FILE, "w") as f:
-        f.write(template)
+    write_file(ZSH_COMPLETION_FILE, template)
 
 
 parser = youtube_dl.parseOpts()[0]
diff --git a/youtube_dl/update.py b/youtube_dl/update.py
index b5f26e4a9..a147b5253 100644
--- a/youtube_dl/update.py
+++ b/youtube_dl/update.py
@@ -1,6 +1,5 @@
 from __future__ import unicode_literals
 
-import io
 import json
 import traceback
 import hashlib

From 0861812d7208310a03909502b1610f5e89d04401 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 25 Jul 2023 15:11:15 +0100
Subject: [PATCH 694/860] [build] Fix typo in `devscripts/fish-completion.py`
 (fix 2285605)

---
 devscripts/fish-completion.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py
index 267ba6a58..ef8a39e0b 100755
--- a/devscripts/fish-completion.py
+++ b/devscripts/fish-completion.py
@@ -43,7 +43,7 @@ def build_completion(opt_parser):
 
     template = read_file(FISH_COMPLETION_TEMPLATE)
     filled_template = template.replace('{{commands}}', '\n'.join(commands))
-    write_file(filled_template)
+    write_file(FISH_COMPLETION_FILE, filled_template)
 
 
 parser = youtube_dl.parseOpts()[0]

From 87e578c9b891b29ab6559ac81ed391897b1e1ace Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 28 Jul 2023 10:52:04 +0100
Subject: [PATCH 695/860] [workflows/ci.yml] Update to setup-java@v3

* avoid Node 12 deprecation
---
 .github/workflows/ci.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 6b91edd6c..a73bedae1 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -278,7 +278,7 @@ jobs:
     #-------- Jython ------
     - name: Set up Java 8
       if: ${{ matrix.python-impl == 'jython' }}
-      uses: actions/setup-java@v2
+      uses: actions/setup-java@v3
       with:
         java-version: 8
         distribution: 'zulu'

From e7926ae9f4e5fa258696551a39295402819280c9 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 28 Jul 2023 06:03:14 +0100
Subject: [PATCH 696/860] [utils] Rework decoding of `Content-Encoding`s

* support nested encodings
* support optional `br` encoding, if brotli package is installed
* support optional 'compress' encoding, if ncompress package is installed
* response `Content-Encoding` has only unprocessed encodings, or removed
* response `Content-Length` is decoded length (usable for filesize metadata)
* use zlib for both deflate and gzip decompression
* some elements taken from yt-dlp: thx especially coletdjnz
---
 test/test_http.py    |  16 ++----
 youtube_dl/compat.py |  14 +++++
 youtube_dl/utils.py  | 120 ++++++++++++++++++++++++++++++++-----------
 3 files changed, 107 insertions(+), 43 deletions(-)

diff --git a/test/test_http.py b/test/test_http.py
index 89580969d..793bea359 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -461,33 +461,23 @@ class TestHTTP(unittest.TestCase):
                 sanitized_Request(
                     self._test_url('content-encoding'),
                     headers={'ytdl-encoding': encoding}))
-            self.assertEqual(res.headers.get('Content-Encoding'), encoding)
+            # decoded encodings are removed: only check for valid decompressed data
             self.assertEqual(res.read(), b'<html><video src="/vid.mp4" /></html>')
 
     @unittest.skipUnless(brotli, 'brotli support is not installed')
-    @unittest.expectedFailure
     def test_brotli(self):
         self.__test_compression('br')
 
-    @unittest.expectedFailure
     def test_deflate(self):
         self.__test_compression('deflate')
 
-    @unittest.expectedFailure
     def test_gzip(self):
         self.__test_compression('gzip')
 
-    @unittest.expectedFailure  # not yet implemented
     def test_multiple_encodings(self):
         # https://www.rfc-editor.org/rfc/rfc9110.html#section-8.4
-        with FakeYDL() as ydl:
-            for pair in ('gzip,deflate', 'deflate, gzip', 'gzip, gzip', 'deflate, deflate'):
-                res = ydl.urlopen(
-                    sanitized_Request(
-                        self._test_url('content-encoding'),
-                        headers={'ytdl-encoding': pair}))
-                self.assertEqual(res.headers.get('Content-Encoding'), pair)
-                self.assertEqual(res.read(), b'<html><video src="/vid.mp4" /></html>')
+        for pair in ('gzip,deflate', 'deflate, gzip', 'gzip, gzip', 'deflate, deflate'):
+            self.__test_compression(pair)
 
     def test_unsupported_encoding(self):
         # it should return the raw content
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index da6d70ec4..54ad64674 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -3200,6 +3200,18 @@ except AttributeError:
     def compat_datetime_timedelta_total_seconds(td):
         return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6
 
+# optional decompression packages
+# PyPi brotli package implements 'br' Content-Encoding
+try:
+    import brotli as compat_brotli
+except ImportError:
+    compat_brotli = None
+# PyPi ncompress package implements 'compress' Content-Encoding
+try:
+    import ncompress as compat_ncompress
+except ImportError:
+    compat_ncompress = None
+
 
 legacy = [
     'compat_HTMLParseError',
@@ -3234,6 +3246,7 @@ __all__ = [
     'compat_Struct',
     'compat_base64_b64decode',
     'compat_basestring',
+    'compat_brotli',
     'compat_casefold',
     'compat_chr',
     'compat_collections_abc',
@@ -3259,6 +3272,7 @@ __all__ = [
     'compat_itertools_zip_longest',
     'compat_kwargs',
     'compat_map',
+    'compat_ncompress',
     'compat_numeric_types',
     'compat_open',
     'compat_ord',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index b5475434f..e73291107 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -15,7 +15,6 @@ import email.utils
 import email.header
 import errno
 import functools
-import gzip
 import inspect
 import io
 import itertools
@@ -42,6 +41,7 @@ from .compat import (
     compat_HTMLParseError,
     compat_HTMLParser,
     compat_basestring,
+    compat_brotli as brotli,
     compat_casefold,
     compat_chr,
     compat_collections_abc,
@@ -55,6 +55,7 @@ from .compat import (
     compat_http_client,
     compat_integer_types,
     compat_kwargs,
+    compat_ncompress as ncompress,
     compat_os_name,
     compat_re_Match,
     compat_re_Pattern,
@@ -2638,11 +2639,44 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
             req)
 
     @staticmethod
-    def deflate(data):
+    def deflate_gz(data):
         try:
-            return zlib.decompress(data, -zlib.MAX_WBITS)
+            # format:zlib,gzip + windowsize:32768
+            return data and zlib.decompress(data, 32 + zlib.MAX_WBITS)
         except zlib.error:
-            return zlib.decompress(data)
+            # raw zlib * windowsize:32768 (RFC 9110: "non-conformant")
+            return zlib.decompress(data, -zlib.MAX_WBITS)
+
+    @staticmethod
+    def gzip(data):
+
+        from gzip import GzipFile
+
+        def _gzip(data):
+            with io.BytesIO(data) as data_buf:
+                gz = GzipFile(fileobj=data_buf, mode='rb')
+                return gz.read()
+
+        try:
+            return _gzip(data)
+        except IOError as original_ioerror:
+            # There may be junk at the end of the file
+            # See http://stackoverflow.com/q/4928560/35070 for details
+            for i in range(1, 1024):
+                try:
+                    return _gzip(data[:-i])
+                except IOError:
+                    continue
+            else:
+                raise original_ioerror
+
+    @staticmethod
+    def brotli(data):
+        return data and brotli.decompress(data)
+
+    @staticmethod
+    def compress(data):
+        return data and ncompress.decompress(data)
 
     def http_request(self, req):
         # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
@@ -2679,33 +2713,59 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
 
     def http_response(self, req, resp):
         old_resp = resp
-        # gzip
-        if resp.headers.get('Content-encoding', '') == 'gzip':
-            content = resp.read()
-            gz = gzip.GzipFile(fileobj=io.BytesIO(content), mode='rb')
-            try:
-                uncompressed = io.BytesIO(gz.read())
-            except IOError as original_ioerror:
-                # There may be junk at the end of the file
-                # See http://stackoverflow.com/q/4928560/35070 for details
-                for i in range(1, 1024):
-                    try:
-                        gz = gzip.GzipFile(fileobj=io.BytesIO(content[:-i]), mode='rb')
-                        uncompressed = io.BytesIO(gz.read())
-                    except IOError:
-                        continue
-                    break
-                else:
-                    raise original_ioerror
-            resp = compat_urllib_request.addinfourl(uncompressed, old_resp.headers, old_resp.url, old_resp.code)
+
+        # Content-Encoding header lists the encodings in order that they were applied [1].
+        # To decompress, we simply do the reverse.
+        # [1]: https://datatracker.ietf.org/doc/html/rfc9110#name-content-encoding
+        decoded_response = None
+        decoders = {
+            'gzip': self.deflate_gz,
+            'deflate': self.deflate_gz,
+        }
+        if brotli:
+            decoders['br'] = self.brotli
+        if ncompress:
+            decoders['compress'] = self.compress
+        if sys.platform.startswith('java'):
+            # Jython zlib implementation misses gzip
+            decoders['gzip'] = self.gzip
+
+        def encodings(hdrs):
+            # A header field that allows multiple values can have multiple instances [2].
+            # [2]: https://datatracker.ietf.org/doc/html/rfc9110#name-fields
+            for e in reversed(','.join(hdrs).split(',')):
+                if e:
+                    yield e.strip()
+
+        encodings_left = []
+        try:
+            resp.headers.get_all
+            hdrs = resp.headers
+        except AttributeError:
+            # Py2 has no get_all() method: headers are rfc822.Message
+            from email.message import Message
+            hdrs = Message()
+            for k, v in resp.headers.items():
+                hdrs[k] = v
+
+        decoder, decoded_response = True, None
+        for encoding in encodings(hdrs.get_all('Content-Encoding', [])):
+            # "SHOULD consider" x-compress, x-gzip as compress, gzip
+            decoder = decoder and decoders.get(remove_start(encoding, 'x-'))
+            if not decoder:
+                encodings_left.insert(0, encoding)
+                continue
+            decoded_response = decoder(decoded_response or resp.read())
+        if decoded_response is not None:
+            resp = compat_urllib_request.addinfourl(
+                io.BytesIO(decoded_response), old_resp.headers, old_resp.url, old_resp.code)
             resp.msg = old_resp.msg
-            del resp.headers['Content-encoding']
-        # deflate
-        if resp.headers.get('Content-encoding', '') == 'deflate':
-            gz = io.BytesIO(self.deflate(resp.read()))
-            resp = compat_urllib_request.addinfourl(gz, old_resp.headers, old_resp.url, old_resp.code)
-            resp.msg = old_resp.msg
-            del resp.headers['Content-encoding']
+            del resp.headers['Content-Length']
+            resp.headers['Content-Length'] = '%d' % len(decoded_response)
+        del resp.headers['Content-Encoding']
+        if encodings_left:
+            resp.headers['Content-Encoding'] = ', '.join(encodings_left)
+
         # Percent-encode redirect URL of Location HTTP header to satisfy RFC 3986 (see
         # https://github.com/ytdl-org/youtube-dl/issues/6457).
         if 300 <= resp.code < 400:

From abef53466da1f7d2e79f5644718a2cf7524abc49 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 28 Jul 2023 06:19:15 +0100
Subject: [PATCH 697/860] [utils] Rework URL path munging for ., .. components

* move processing to YoutubeDLHandler
* also process `Location` header for redirect
* use tests from https://github.com/yt-dlp/yt-dlp/pull/7662
---
 test/test_http.py       | 14 +++++++++
 test/test_utils.py      | 29 ++++++++++++++++-
 youtube_dl/YoutubeDL.py | 23 --------------
 youtube_dl/utils.py     | 70 +++++++++++++++++++++++++++++++----------
 4 files changed, 95 insertions(+), 41 deletions(-)

diff --git a/test/test_http.py b/test/test_http.py
index 793bea359..485c4c6fc 100644
--- a/test/test_http.py
+++ b/test/test_http.py
@@ -180,6 +180,12 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
             respond()
         elif self.path == '/%c7%9f':
             respond()
+        elif self.path == '/redirect_dotsegments':
+            self.send_response(301)
+            # redirect to /headers but with dot segments before
+            self.send_header('Location', '/a/b/./../../headers')
+            self.send_header('Content-Length', '0')
+            self.end_headers()
         elif self.path.startswith('/redirect_'):
             self._redirect()
         elif self.path.startswith('/method'):
@@ -489,6 +495,14 @@ class TestHTTP(unittest.TestCase):
             self.assertEqual(res.headers.get('Content-Encoding'), 'unsupported')
             self.assertEqual(res.read(), b'raw')
 
+    def test_remove_dot_segments(self):
+        with FakeYDL() as ydl:
+            res = ydl.urlopen(sanitized_Request(self._test_url('a/b/./../../headers')))
+            self.assertEqual(compat_urllib_parse.urlparse(res.geturl()).path, '/headers')
+
+            res = ydl.urlopen(sanitized_Request(self._test_url('redirect_dotsegments')))
+            self.assertEqual(compat_urllib_parse.urlparse(res.geturl()).path, '/headers')
+
 
 def _build_proxy_handler(name):
     class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler):
diff --git a/test/test_utils.py b/test/test_utils.py
index e83977f29..fdae1f744 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -64,6 +64,7 @@ from youtube_dl.utils import (
     parse_age_limit,
     parse_duration,
     parse_filesize,
+    parse_codecs,
     parse_count,
     parse_iso8601,
     parse_resolution,
@@ -114,7 +115,7 @@ from youtube_dl.utils import (
     cli_option,
     cli_valueless_option,
     cli_bool_option,
-    parse_codecs,
+    YoutubeDLHandler,
 )
 from youtube_dl.compat import (
     compat_chr,
@@ -905,6 +906,32 @@ class TestUtil(unittest.TestCase):
         )
         self.assertEqual(escape_url('http://vimeo.com/56015672#at=0'), 'http://vimeo.com/56015672#at=0')
 
+    def test_remove_dot_segments(self):
+
+        def remove_dot_segments(p):
+            q = '' if p.startswith('/') else '/'
+            p = 'http://example.com' + q + p
+            p = compat_urlparse.urlsplit(YoutubeDLHandler._fix_path(p)).path
+            return p[1:] if q else p
+
+        self.assertEqual(remove_dot_segments('/a/b/c/./../../g'), '/a/g')
+        self.assertEqual(remove_dot_segments('mid/content=5/../6'), 'mid/6')
+        self.assertEqual(remove_dot_segments('/ad/../cd'), '/cd')
+        self.assertEqual(remove_dot_segments('/ad/../cd/'), '/cd/')
+        self.assertEqual(remove_dot_segments('/..'), '/')
+        self.assertEqual(remove_dot_segments('/./'), '/')
+        self.assertEqual(remove_dot_segments('/./a'), '/a')
+        self.assertEqual(remove_dot_segments('/abc/./.././d/././e/.././f/./../../ghi'), '/ghi')
+        self.assertEqual(remove_dot_segments('/'), '/')
+        self.assertEqual(remove_dot_segments('/t'), '/t')
+        self.assertEqual(remove_dot_segments('t'), 't')
+        self.assertEqual(remove_dot_segments(''), '')
+        self.assertEqual(remove_dot_segments('/../a/b/c'), '/a/b/c')
+        self.assertEqual(remove_dot_segments('../a'), 'a')
+        self.assertEqual(remove_dot_segments('./a'), 'a')
+        self.assertEqual(remove_dot_segments('.'), '')
+        self.assertEqual(remove_dot_segments('////'), '////')
+
     def test_js_to_json_vars_strings(self):
         self.assertDictEqual(
             json.loads(js_to_json(
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6a12f91e4..13a41928f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -71,7 +71,6 @@ from .utils import (
     format_bytes,
     formatSeconds,
     GeoRestrictedError,
-    HEADRequest,
     int_or_none,
     ISO3166Utils,
     join_nonempty,
@@ -88,7 +87,6 @@ from .utils import (
     preferredencoding,
     prepend_extension,
     process_communicate_or_kill,
-    PUTRequest,
     register_socks_protocols,
     render_table,
     replace_extension,
@@ -2460,27 +2458,6 @@ class YoutubeDL(object):
         """ Start an HTTP download """
         if isinstance(req, compat_basestring):
             req = sanitized_Request(req)
-        # an embedded /../ sequence is not automatically handled by urllib2
-        # see https://github.com/yt-dlp/yt-dlp/issues/3355
-        url = req.get_full_url()
-        parts = url.partition('/../')
-        if parts[1]:
-            url = compat_urllib_parse.urljoin(parts[0] + parts[1][:1], parts[1][1:] + parts[2])
-        if url:
-            # worse, URL path may have initial /../ against RFCs: work-around
-            # by stripping such prefixes, like eg Firefox
-            parts = compat_urllib_parse.urlsplit(url)
-            path = parts.path
-            while path.startswith('/../'):
-                path = path[3:]
-            url = parts._replace(path=path).geturl()
-            # get a new Request with the munged URL
-            if url != req.get_full_url():
-                req_type = {'HEAD': HEADRequest, 'PUT': PUTRequest}.get(
-                    req.get_method(), compat_urllib_request.Request)
-                req = req_type(
-                    url, data=req.data, headers=dict(req.header_items()),
-                    origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)
         return self._opener.open(req, timeout=self._socket_timeout)
 
     def print_debug_header(self):
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e73291107..36204c8fa 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2678,17 +2678,52 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
     def compress(data):
         return data and ncompress.decompress(data)
 
+    @staticmethod
+    def _fix_path(url):
+        # an embedded /../ or /./ sequence is not automatically handled by urllib2
+        # see https://github.com/yt-dlp/yt-dlp/issues/3355
+        parsed_url = compat_urllib_parse.urlsplit(url)
+        path = parsed_url.path
+        if not path.endswith('/'):
+            path += '/'
+        parts = path.partition('/./')
+        if not parts[1]:
+            parts = path.partition('/../')
+        if parts[1]:
+            path = compat_urllib_parse.urljoin(
+                parts[0] + parts[1][:1],
+                parts[1][1:] + (parts[2] if parsed_url.path.endswith('/') else parts[2][:-1]))
+            url = parsed_url._replace(path=path).geturl()
+        if '/.' in url:
+            # worse, URL path may have initial /../ against RFCs: work-around
+            # by stripping such prefixes, like eg Firefox
+            path = parsed_url.path + '/'
+            while path.startswith('/.'):
+                if path.startswith('/../'):
+                    path = path[3:]
+                elif path.startswith('/./'):
+                    path = path[2:]
+                else:
+                    break
+            path = path[:-1]
+            if not path.startswith('/') and parsed_url.path.startswith('/'):
+                path = '/' + path
+            url = parsed_url._replace(path=path).geturl()
+        return url
+
     def http_request(self, req):
-        # According to RFC 3986, URLs can not contain non-ASCII characters, however this is not
-        # always respected by websites, some tend to give out URLs with non percent-encoded
+        url = req.get_full_url()
+        # resolve embedded . and ..
+        url_fixed = self._fix_path(url)
+        # According to RFC 3986, URLs can not contain non-ASCII characters; however this is not
+        # always respected by websites: some tend to give out URLs with non percent-encoded
         # non-ASCII characters (see telemb.py, ard.py [#3412])
         # urllib chokes on URLs with non-ASCII characters (see http://bugs.python.org/issue3991)
         # To work around aforementioned issue we will replace request's original URL with
         # percent-encoded one
         # Since redirects are also affected (e.g. http://www.southpark.de/alle-episoden/s18e09)
         # the code of this workaround has been moved here from YoutubeDL.urlopen()
-        url = req.get_full_url()
-        url_escaped = escape_url(url)
+        url_escaped = escape_url(url_fixed)
 
         # Substitute URL if any change after escaping
         if url != url_escaped:
@@ -2702,10 +2737,13 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
 
         req.headers = handle_youtubedl_headers(req.headers)
 
-        if sys.version_info < (2, 7) and '#' in req.get_full_url():
-            # Python 2.6 is brain-dead when it comes to fragments
-            req._Request__original = req._Request__original.partition('#')[0]
-            req._Request__r_type = req._Request__r_type.partition('#')[0]
+        if sys.version_info < (2, 7):
+            # avoid possible race where __r_type may be unset
+            req.get_type()
+            if '#' in req.get_full_url():
+                # Python 2.6 is brain-dead when it comes to fragments
+                req._Request__original = req._Request__original.partition('#')[0]
+                req._Request__r_type = req._Request__r_type.partition('#')[0]
 
         # Use the totally undocumented AbstractHTTPHandler per
         # https://github.com/yt-dlp/yt-dlp/pull/4158
@@ -2775,10 +2813,13 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
                 if sys.version_info >= (3, 0):
                     location = location.encode('iso-8859-1')
                 location = location.decode('utf-8')
-                location_escaped = escape_url(location)
+                # resolve embedded . and ..
+                location_fixed = self._fix_path(location)
+                location_escaped = escape_url(location_fixed)
                 if location != location_escaped:
                     del resp.headers['Location']
-                    if sys.version_info < (3, 0):
+                    # if sys.version_info < (3, 0):
+                    if not isinstance(location_escaped, str):
                         location_escaped = location_escaped.encode('utf-8')
                     resp.headers['Location'] = location_escaped
         return resp
@@ -4248,13 +4289,8 @@ def update_Request(req, url=None, data=None, headers={}, query={}):
     req_headers.update(headers)
     req_data = data if data is not None else req.data
     req_url = update_url_query(url or req.get_full_url(), query)
-    req_get_method = req.get_method()
-    if req_get_method == 'HEAD':
-        req_type = HEADRequest
-    elif req_get_method == 'PUT':
-        req_type = PUTRequest
-    else:
-        req_type = compat_urllib_request.Request
+    req_type = {'HEAD': HEADRequest, 'PUT': PUTRequest}.get(
+        req.get_method(), compat_urllib_request.Request)
     new_req = req_type(
         req_url, data=req_data, headers=req_headers,
         origin_req_host=req.origin_req_host, unverifiable=req.unverifiable)

From 7d965e6b65655f2a5fbae34219fc87359a3d7061 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 30 Jul 2023 21:45:57 +0100
Subject: [PATCH 698/860] [utils] Avoid comparing `type(var)`, etc, to pass new
 Linter rules

---
 youtube_dl/swfinterp.py |  2 +-
 youtube_dl/utils.py     | 10 ++++++----
 2 files changed, 7 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/swfinterp.py b/youtube_dl/swfinterp.py
index 0c7158575..e79e0b17f 100644
--- a/youtube_dl/swfinterp.py
+++ b/youtube_dl/swfinterp.py
@@ -727,7 +727,7 @@ class SWFInterpreter(object):
                             stack.append(res)
                             continue
 
-                        assert isinstance(obj, (dict, _ScopeDict)),\
+                        assert isinstance(obj, (dict, _ScopeDict)), \
                             'Accessing member %r on %r' % (pname, obj)
                         res = obj.get(pname, undefined)
                         stack.append(res)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 36204c8fa..1da5a7a38 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2235,7 +2235,7 @@ def _htmlentity_transform(entity_with_semicolon):
 def unescapeHTML(s):
     if s is None:
         return None
-    assert type(s) == compat_str
+    assert isinstance(s, compat_str)
 
     return re.sub(
         r'&([^&;]+;)', lambda m: _htmlentity_transform(m.group(1)), s)
@@ -3418,7 +3418,7 @@ def _windows_write_string(s, out):
 def write_string(s, out=None, encoding=None):
     if out is None:
         out = sys.stderr
-    assert type(s) == compat_str
+    assert isinstance(s, compat_str)
 
     if sys.platform == 'win32' and encoding is None and hasattr(out, 'fileno'):
         if _windows_write_string(s, out):
@@ -4459,8 +4459,10 @@ TV_PARENTAL_GUIDELINES = {
 
 
 def parse_age_limit(s):
-    if type(s) == int:
-        return s if 0 <= s <= 21 else None
+    if not isinstance(s, bool):
+        age = int_or_none(s)
+        if age is not None:
+            return age if 0 <= age <= 21 else None
     if not isinstance(s, compat_basestring):
         return None
     m = re.match(r'^(?P<age>\d{1,2})\+?$', s)

From 2d2a4bc8324fc4bc5a235cbd1ee0b0769912bfd1 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 30 Jul 2023 21:47:48 +0100
Subject: [PATCH 699/860] [utils] Revise `isinstance()` tests (especially for
 str/unicode/bytes) to complete Linter fix

---
 youtube_dl/compat.py |   2 +-
 youtube_dl/utils.py  | 153 ++++++++++++++++++++-----------------------
 2 files changed, 73 insertions(+), 82 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 54ad64674..3c526a78d 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -36,7 +36,7 @@ try:
     )
 except NameError:
     compat_str, compat_basestring, compat_chr = (
-        str, str, chr
+        str, (str, bytes), chr
     )
 
 # casefold
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1da5a7a38..94b339b1d 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1826,11 +1826,11 @@ def write_json_file(obj, fn):
     if sys.version_info < (3, 0) and sys.platform != 'win32':
         encoding = get_filesystem_encoding()
         # os.path.basename returns a bytes object, but NamedTemporaryFile
-        # will fail if the filename contains non ascii characters unless we
+        # will fail if the filename contains non-ascii characters unless we
         # use a unicode object
-        path_basename = lambda f: os.path.basename(fn).decode(encoding)
+        path_basename = lambda f: os.path.basename(f).decode(encoding)
         # the same for os.path.dirname
-        path_dirname = lambda f: os.path.dirname(fn).decode(encoding)
+        path_dirname = lambda f: os.path.dirname(f).decode(encoding)
     else:
         path_basename = os.path.basename
         path_dirname = os.path.dirname
@@ -1894,10 +1894,10 @@ else:
                 return f
         return None
 
+
 # On python2.6 the xml.etree.ElementTree.Element methods don't support
 # the namespace parameter
 
-
 def xpath_with_ns(path, ns_map):
     components = [c.split(':') for c in path.split('/')]
     replaced = []
@@ -1914,7 +1914,7 @@ def xpath_element(node, xpath, name=None, fatal=False, default=NO_DEFAULT):
     def _find_xpath(xpath):
         return node.find(compat_xpath(xpath))
 
-    if isinstance(xpath, (str, compat_str)):
+    if isinstance(xpath, compat_basestring):
         n = _find_xpath(xpath)
     else:
         for xp in xpath:
@@ -2262,39 +2262,32 @@ def get_subprocess_encoding():
     return encoding
 
 
-def encodeFilename(s, for_subprocess=False):
-    """
-    @param s The name of the file
-    """
+# Jython assumes filenames are Unicode strings though reported as Python 2.x compatible
+if sys.version_info < (3, 0) and not sys.platform.startswith('java'):
 
-    assert type(s) == compat_str
+    def encodeFilename(s, for_subprocess=False):
+        """
+        @param s The name of the file
+        """
+
+        # Pass '' directly to use Unicode APIs on Windows 2000 and up
+        # (Detecting Windows NT 4 is tricky because 'major >= 4' would
+        # match Windows 9x series as well. Besides, NT 4 is obsolete.)
+        if (not for_subprocess
+                and sys.platform == 'win32'
+                and sys.getwindowsversion()[0] >= 5
+                and isinstance(s, compat_str)):
+            return s
+
+        return _encode_compat_str(s, get_subprocess_encoding(), 'ignore')
+
+    def decodeFilename(b, for_subprocess=False):
+        return _decode_compat_str(b, get_subprocess_encoding(), 'ignore')
+
+else:
 
     # Python 3 has a Unicode API
-    if sys.version_info >= (3, 0):
-        return s
-
-    # Pass '' directly to use Unicode APIs on Windows 2000 and up
-    # (Detecting Windows NT 4 is tricky because 'major >= 4' would
-    # match Windows 9x series as well. Besides, NT 4 is obsolete.)
-    if not for_subprocess and sys.platform == 'win32' and sys.getwindowsversion()[0] >= 5:
-        return s
-
-    # Jython assumes filenames are Unicode strings though reported as Python 2.x compatible
-    if sys.platform.startswith('java'):
-        return s
-
-    return s.encode(get_subprocess_encoding(), 'ignore')
-
-
-def decodeFilename(b, for_subprocess=False):
-
-    if sys.version_info >= (3, 0):
-        return b
-
-    if not isinstance(b, bytes):
-        return b
-
-    return b.decode(get_subprocess_encoding(), 'ignore')
+    encodeFilename = decodeFilename = lambda *s, **k: s[0]
 
 
 def encodeArgument(s):
@@ -2313,11 +2306,7 @@ def decodeArgument(b):
 def decodeOption(optval):
     if optval is None:
         return optval
-    if isinstance(optval, bytes):
-        optval = optval.decode(preferredencoding())
-
-    assert isinstance(optval, compat_str)
-    return optval
+    return _decode_compat_str(optval)
 
 
 def formatSeconds(secs):
@@ -2363,7 +2352,7 @@ def make_HTTPS_handler(params, **kwargs):
 
     if sys.version_info < (3, 2):
         return YoutubeDLHTTPSHandler(params, **kwargs)
-    else:  # Python < 3.4
+    else:  # Python3 < 3.4
         context = ssl.SSLContext(ssl.PROTOCOL_TLSv1)
         context.verify_mode = (ssl.CERT_NONE
                                if opts_no_check_certificate
@@ -2818,8 +2807,7 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
                 location_escaped = escape_url(location_fixed)
                 if location != location_escaped:
                     del resp.headers['Location']
-                    # if sys.version_info < (3, 0):
-                    if not isinstance(location_escaped, str):
+                    if not isinstance(location_escaped, str):  # Py 2 case
                         location_escaped = location_escaped.encode('utf-8')
                     resp.headers['Location'] = location_escaped
         return resp
@@ -3086,8 +3074,7 @@ class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
         # On python 2 urlh.geturl() may sometimes return redirect URL
         # as a byte string instead of unicode. This workaround forces
         # it to return unicode.
-        if sys.version_info[0] < 3:
-            newurl = compat_str(newurl)
+        newurl = _decode_compat_str(newurl)
 
         # Be conciliant with URIs containing a space.  This is mainly
         # redundant with the more complete encoding done in http_error_302(),
@@ -3333,11 +3320,7 @@ class DateRange(object):
 def platform_name():
     """ Returns the platform name as a compat_str """
     res = platform.platform()
-    if isinstance(res, bytes):
-        res = res.decode(preferredencoding())
-
-    assert isinstance(res, compat_str)
-    return res
+    return _decode_compat_str(res)
 
 
 def _windows_write_string(s, out):
@@ -3567,9 +3550,8 @@ def shell_quote(args):
     quoted_args = []
     encoding = get_filesystem_encoding()
     for a in args:
-        if isinstance(a, bytes):
-            # We may get a filename encoded with 'encodeFilename'
-            a = a.decode(encoding)
+        # We may get a filename encoded with 'encodeFilename'
+        a = _decode_compat_str(a, encoding)
         quoted_args.append(compat_shlex_quote(a))
     return ' '.join(quoted_args)
 
@@ -3733,8 +3715,9 @@ def parse_resolution(s):
 
 
 def parse_bitrate(s):
-    if not isinstance(s, compat_str):
-        return
+    s = txt_or_none(s)
+    if not s:
+        return None
     mobj = re.search(r'\b(\d+)\s*kbps', s)
     if mobj:
         return int(mobj.group(1))
@@ -3822,18 +3805,17 @@ def base_url(url):
 
 
 def urljoin(base, path):
-    if isinstance(path, bytes):
-        path = path.decode('utf-8')
-    if not isinstance(path, compat_str) or not path:
+    path = _decode_compat_str(path, encoding='utf-8', or_none=True)
+    if not path:
         return None
     if re.match(r'^(?:[a-zA-Z][a-zA-Z0-9+-.]*:)?//', path):
         return path
-    if isinstance(base, bytes):
-        base = base.decode('utf-8')
-    if not isinstance(base, compat_str) or not re.match(
-            r'^(?:https?:)?//', base):
+    base = _decode_compat_str(base, encoding='utf-8', or_none=True)
+    if not base:
         return None
-    return compat_urllib_parse.urljoin(base, path)
+    return (
+        re.match(r'^(?:https?:)?//', base)
+        and compat_urllib_parse.urljoin(base, path))
 
 
 class HEADRequest(compat_urllib_request.Request):
@@ -3998,8 +3980,7 @@ def get_exe_version(exe, args=['--version'],
             stdout=subprocess.PIPE, stderr=subprocess.STDOUT))
     except OSError:
         return False
-    if isinstance(out, bytes):  # Python 2.x
-        out = out.decode('ascii', 'ignore')
+    out = _decode_compat_str(out, 'ascii', 'ignore')
     return detect_exe_version(out, version_re, unrecognized)
 
 
@@ -4218,8 +4199,8 @@ def lowercase_escape(s):
 
 def escape_rfc3986(s):
     """Escape non-ASCII characters as suggested by RFC 3986"""
-    if sys.version_info < (3, 0) and isinstance(s, compat_str):
-        s = s.encode('utf-8')
+    if sys.version_info < (3, 0):
+        s = _encode_compat_str(s, 'utf-8')
     # ensure unicode: after quoting, it can always be converted
     return compat_str(compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]"))
 
@@ -4242,8 +4223,7 @@ def parse_qs(url, **kwargs):
 
 def read_batch_urls(batch_fd):
     def fixup(url):
-        if not isinstance(url, compat_str):
-            url = url.decode('utf-8', 'replace')
+        url = _decode_compat_str(url, 'utf-8', 'replace')
         BOM_UTF8 = '\xef\xbb\xbf'
         if url.startswith(BOM_UTF8):
             url = url[len(BOM_UTF8):]
@@ -4305,10 +4285,8 @@ def _multipart_encode_impl(data, boundary):
     out = b''
     for k, v in data.items():
         out += b'--' + boundary.encode('ascii') + b'\r\n'
-        if isinstance(k, compat_str):
-            k = k.encode('utf-8')
-        if isinstance(v, compat_str):
-            v = v.encode('utf-8')
+        k = _encode_compat_str(k, 'utf-8')
+        v = _encode_compat_str(v, 'utf-8')
         # RFC 2047 requires non-ASCII field names to be encoded, while RFC 7578
         # suggests sending UTF-8 directly. Firefox sends UTF-8, too
         content = b'Content-Disposition: form-data; name="' + k + b'"\r\n\r\n' + v + b'\r\n'
@@ -4435,8 +4413,26 @@ def merge_dicts(*dicts, **kwargs):
     return merged
 
 
-def encode_compat_str(string, encoding=preferredencoding(), errors='strict'):
-    return string if isinstance(string, compat_str) else compat_str(string, encoding, errors)
+# very poor choice of name, as if Python string encodings weren't confusing enough
+def encode_compat_str(s, encoding=preferredencoding(), errors='strict'):
+    assert isinstance(s, compat_basestring)
+    return s if isinstance(s, compat_str) else compat_str(s, encoding, errors)
+
+
+# what it could have been
+def _decode_compat_str(s, encoding=preferredencoding(), errors='strict', or_none=False):
+    if not or_none:
+        assert isinstance(s, compat_basestring)
+    return (
+        s if isinstance(s, compat_str)
+        else compat_str(s, encoding, errors) if isinstance(s, compat_basestring)
+        else None)
+
+
+# the real encode_compat_str, but only for internal use
+def _encode_compat_str(s, encoding=preferredencoding(), errors='strict'):
+    assert isinstance(s, compat_basestring)
+    return s.encode(encoding, errors) if isinstance(s, compat_str) else s
 
 
 US_RATINGS = {
@@ -4639,12 +4635,7 @@ def args_to_str(args):
 
 
 def error_to_compat_str(err):
-    err_str = str(err)
-    # On python 2 error byte string must be decoded with proper
-    # encoding rather than ascii
-    if sys.version_info[0] < 3:
-        err_str = err_str.decode(preferredencoding())
-    return err_str
+    return _decode_compat_str(str(err))
 
 
 def mimetype2ext(mt):

From e4178b5af3428f29feca622d531090f10f54af35 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 30 Jul 2023 21:49:58 +0100
Subject: [PATCH 700/860] [utils] Add and use `filter_dict()` from yt-dlp

---
 youtube_dl/utils.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 94b339b1d..c530ed5a2 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2586,7 +2586,7 @@ def handle_youtubedl_headers(headers):
     filtered_headers = headers
 
     if 'Youtubedl-no-compression' in filtered_headers:
-        filtered_headers = dict((k, v) for k, v in filtered_headers.items() if k.lower() != 'accept-encoding')
+        filtered_headers = filter_dict(filtered_headers, cndn=lambda k, _: k.lower() != 'accept-encoding')
         del filtered_headers['Youtubedl-no-compression']
 
     return filtered_headers
@@ -3102,9 +3102,7 @@ class YoutubeDLRedirectHandler(compat_urllib_request.HTTPRedirectHandler):
             new_data = None
             remove_headers.extend(['Content-Length', 'Content-Type'])
 
-        # NB: don't use dict comprehension for python 2.6 compatibility
-        new_headers = dict((k, v) for k, v in req.headers.items()
-                           if k.title() not in remove_headers)
+        new_headers = filter_dict(req.headers, cndn=lambda k, _: k.title() not in remove_headers)
 
         return compat_urllib_request.Request(
             newurl, headers=new_headers, origin_req_host=req.origin_req_host,
@@ -4377,6 +4375,11 @@ def try_get(src, getter, expected_type=None):
                 return v
 
 
+def filter_dict(dct, cndn=lambda _, v: v is not None):
+    # NB: don't use dict comprehension for python 2.6 compatibility
+    return dict((k, v) for k, v in dct.items() if cndn(k, v))
+
+
 def merge_dicts(*dicts, **kwargs):
     """
         Merge the `dict`s in `dicts` using the first valid value for each key.

From 2efc8de4d2299e08e0c84d674d7fc7f3fa669487 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 30 Jul 2023 21:50:52 +0100
Subject: [PATCH 701/860] [utils] Advertise optional supported
 `Content-Encoding`s

---
 youtube_dl/utils.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c530ed5a2..81ff78807 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1678,9 +1678,7 @@ def random_user_agent():
 
 std_headers = {
     'User-Agent': random_user_agent(),
-    'Accept-Charset': 'ISO-8859-1,utf-8;q=0.7,*;q=0.7',
     'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8',
-    'Accept-Encoding': 'gzip, deflate',
     'Accept-Language': 'en-us,en;q=0.5',
 }
 
@@ -2724,6 +2722,13 @@ class YoutubeDLHandler(compat_urllib_request.HTTPHandler):
             if h.capitalize() not in req.headers:
                 req.add_header(h, v)
 
+        # Similarly, 'Accept-encoding'
+        if 'Accept-encoding' not in req.headers:
+            req.add_header(
+                'Accept-Encoding', join_nonempty(
+                    'gzip', 'deflate', brotli and 'br', ncompress and 'compress',
+                    delim=', '))
+
         req.headers = handle_youtubedl_headers(req.headers)
 
         if sys.version_info < (2, 7):

From 86e3cf5e5849aefcc540c19bb5fa5ab7f470d1c1 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Aug 2023 22:54:12 +0100
Subject: [PATCH 702/860] [S4C] Add extractor for Sianel Pedwar Cymru

* from https://github.com/yt-dlp/yt-dlp/pull/7730, thx ifan-t, bashonly
---
 youtube_dl/extractor/extractors.py |  1 +
 youtube_dl/extractor/s4c.py        | 76 ++++++++++++++++++++++++++++++
 2 files changed, 77 insertions(+)
 create mode 100644 youtube_dl/extractor/s4c.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 42b009ef5..cb39876c2 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1087,6 +1087,7 @@ from .rutube import (
 from .rutv import RUTVIE
 from .ruutu import RuutuIE
 from .ruv import RuvIE
+from .s4c import S4CIE
 from .safari import (
     SafariIE,
     SafariApiIE,
diff --git a/youtube_dl/extractor/s4c.py b/youtube_dl/extractor/s4c.py
new file mode 100644
index 000000000..21d40c2d3
--- /dev/null
+++ b/youtube_dl/extractor/s4c.py
@@ -0,0 +1,76 @@
+# coding: utf-8
+
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    merge_dicts,
+    T,
+    traverse_obj,
+    txt_or_none,
+)
+
+
+class S4CIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?s4c\.cymru/clic/programme/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.s4c.cymru/clic/programme/861362209',
+        'info_dict': {
+            'id': '861362209',
+            'ext': 'mp4',
+            'title': 'Y Swn',
+            'description': 'md5:f7681a30e4955b250b3224aa9fe70cf0',
+            'duration': 5340
+        },
+    }, {
+        'url': 'https://www.s4c.cymru/clic/programme/856636948',
+        'info_dict': {
+            'id': '856636948',
+            'ext': 'mp4',
+            'title': 'Am Dro',
+            'duration': 2880,
+            'description': 'md5:100d8686fc9a632a0cb2db52a3433ffe',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        details = self._download_json(
+            'https://www.s4c.cymru/df/full_prog_details',
+            video_id, query={
+                'lang': 'e',
+                'programme_id': video_id,
+            }, fatal=False)
+
+        filename = self._download_json(
+            'https://player-api.s4c-cdn.co.uk/player-configuration/prod', video_id, query={
+                'programme_id': video_id,
+                'signed': '0',
+                'lang': 'en',
+                'mode': 'od',
+                'appId': 'clic',
+                'streamName': '',
+            }, note='Downloading player config JSON')['filename']
+        m3u8_url = self._download_json(
+            'https://player-api.s4c-cdn.co.uk/streaming-urls/prod', video_id, query={
+                'mode': 'od',
+                'application': 'clic',
+                'region': 'WW',
+                'extra': 'false',
+                'thirdParty': 'false',
+                'filename': filename,
+            }, note='Downloading streaming urls JSON')['hls']
+        # ... self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls')
+        formats, subtitles = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', m3u8_id='hls', entry_protocol='m3u8_native'), {}
+
+        return merge_dicts({
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+        }, traverse_obj(details, ('full_prog_details', 0, {
+            'title': (('programme_title', 'series_title'), T(txt_or_none)),
+            'description': ('full_billing', T(txt_or_none)),
+            'duration': ('duration', T(lambda x: float_or_none(x, invscale=60))),
+        }), get_all=False),
+            rev=True)

From 7d58f0769a8f08e46ea77432041577cef94c07e2 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 31 Aug 2023 17:16:47 +0100
Subject: [PATCH 703/860] [ci.yml] Improve conditions for nosetest
 installations

---
 .github/workflows/ci.yml | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index a73bedae1..7fb8f9f83 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -368,7 +368,7 @@ jobs:
         done
     #-------- nose --------
     - name: Install nose for Python ${{ matrix.python-version }}
-      if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.7' || matrix.python-version == '3.12' }}
+      if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || (matrix.python-impl == 'cpython' && (matrix.python-version == '2.7' || matrix.python-version == '3.12')) }}
       shell: bash
       run: |
         echo "$PATH"
@@ -380,7 +380,7 @@ jobs:
         [ "$py3ver" -ge 9 ] && nose=pynose || nose=nose
         $PIP -qq show $nose || $PIP install $nose
     - name: Install nose for other Python 2
-      if: ${{ matrix.python-impl == 'jython' || matrix.python-version == '2.6' }}
+      if: ${{ matrix.python-impl == 'jython' || (matrix.python-impl == 'cpython' && matrix.python-version == '2.6') }}
       shell: bash
       run: |
         # Work around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb)

From 31f50c8194f12c27ac6fbfe336f1d515aa8677ae Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 27 Aug 2023 19:08:28 +0100
Subject: [PATCH 704/860] [S4C] Add thumbnail extraction, extract series as
 playlist

Based on https://github.com/yt-dlp/yt-dlp/pull/7776: thx ifan-t, bashonly
---
 youtube_dl/extractor/extractors.py |  5 ++-
 youtube_dl/extractor/s4c.py        | 62 ++++++++++++++++++++++++++----
 2 files changed, 59 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index cb39876c2..d9289e5bf 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1087,7 +1087,10 @@ from .rutube import (
 from .rutv import RUTVIE
 from .ruutu import RuutuIE
 from .ruv import RuvIE
-from .s4c import S4CIE
+from .s4c import (
+    S4CIE,
+    S4CSeriesIE,
+)
 from .safari import (
     SafariIE,
     SafariApiIE,
diff --git a/youtube_dl/extractor/s4c.py b/youtube_dl/extractor/s4c.py
index 21d40c2d3..b152e6680 100644
--- a/youtube_dl/extractor/s4c.py
+++ b/youtube_dl/extractor/s4c.py
@@ -2,6 +2,8 @@
 
 from __future__ import unicode_literals
 
+from functools import partial as partial_f
+
 from .common import InfoExtractor
 from ..utils import (
     float_or_none,
@@ -9,6 +11,7 @@ from ..utils import (
     T,
     traverse_obj,
     txt_or_none,
+    url_or_none,
 )
 
 
@@ -21,7 +24,8 @@ class S4CIE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Y Swn',
             'description': 'md5:f7681a30e4955b250b3224aa9fe70cf0',
-            'duration': 5340
+            'duration': 5340,
+            'thumbnail': 'https://www.s4c.cymru/amg/1920x1080/Y_Swn_2023S4C_099_ii.jpg',
         },
     }, {
         'url': 'https://www.s4c.cymru/clic/programme/856636948',
@@ -31,6 +35,7 @@ class S4CIE(InfoExtractor):
             'title': 'Am Dro',
             'duration': 2880,
             'description': 'md5:100d8686fc9a632a0cb2db52a3433ffe',
+            'thumbnail': 'https://www.s4c.cymru/amg/1920x1080/Am_Dro_2022-23S4C_P6_4005.jpg',
         },
     }]
 
@@ -43,7 +48,7 @@ class S4CIE(InfoExtractor):
                 'programme_id': video_id,
             }, fatal=False)
 
-        filename = self._download_json(
+        player_config = self._download_json(
             'https://player-api.s4c-cdn.co.uk/player-configuration/prod', video_id, query={
                 'programme_id': video_id,
                 'signed': '0',
@@ -51,7 +56,8 @@ class S4CIE(InfoExtractor):
                 'mode': 'od',
                 'appId': 'clic',
                 'streamName': '',
-            }, note='Downloading player config JSON')['filename']
+            }, note='Downloading player config JSON')
+
         m3u8_url = self._download_json(
             'https://player-api.s4c-cdn.co.uk/streaming-urls/prod', video_id, query={
                 'mode': 'od',
@@ -59,18 +65,60 @@ class S4CIE(InfoExtractor):
                 'region': 'WW',
                 'extra': 'false',
                 'thirdParty': 'false',
-                'filename': filename,
+                'filename': player_config['filename'],
             }, note='Downloading streaming urls JSON')['hls']
-        # ... self._extract_m3u8_formats_and_subtitles(m3u8_url, video_id, 'mp4', m3u8_id='hls')
-        formats, subtitles = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', m3u8_id='hls', entry_protocol='m3u8_native'), {}
+        formats = self._extract_m3u8_formats(m3u8_url, video_id, 'mp4', m3u8_id='hls', entry_protocol='m3u8_native')
+        self._sort_formats(formats)
+
+        subtitles = {}
+        for sub in traverse_obj(player_config, ('subtitles', lambda _, v: url_or_none(v['0']))):
+            subtitles.setdefault(sub.get('3', 'en'), []).append({
+                'url': sub['0'],
+                'name': sub.get('1'),
+            })
 
         return merge_dicts({
             'id': video_id,
             'formats': formats,
             'subtitles': subtitles,
+            'thumbnail': url_or_none(player_config.get('poster')),
         }, traverse_obj(details, ('full_prog_details', 0, {
             'title': (('programme_title', 'series_title'), T(txt_or_none)),
             'description': ('full_billing', T(txt_or_none)),
-            'duration': ('duration', T(lambda x: float_or_none(x, invscale=60))),
+            'duration': ('duration', T(partial_f(float_or_none, invscale=60))),
         }), get_all=False),
             rev=True)
+
+
+class S4CSeriesIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?s4c\.cymru/clic/series/(?P<id>\d+)'
+    _TESTS = [{
+        'url': 'https://www.s4c.cymru/clic/series/864982911',
+        'playlist_mincount': 6,
+        'info_dict': {
+            'id': '864982911',
+            'title': 'Iaith ar Daith',
+        },
+    }, {
+        'url': 'https://www.s4c.cymru/clic/series/866852587',
+        'playlist_mincount': 8,
+        'info_dict': {
+            'id': '866852587',
+            'title': 'FFIT Cymru',
+        },
+    }]
+
+    def _real_extract(self, url):
+        series_id = self._match_id(url)
+        series_details = self._download_json(
+            'https://www.s4c.cymru/df/series_details', series_id, query={
+                'lang': 'e',
+                'series_id': series_id,
+                'show_prog_in_series': 'Y'
+            }, note='Downloading series details JSON')
+
+        return self.playlist_result(
+            (self.url_result('https://www.s4c.cymru/clic/programme/' + episode_id, S4CIE, episode_id)
+             for episode_id in traverse_obj(series_details, ('other_progs_in_series', Ellipsis, 'id'))),
+            playlist_id=series_id, playlist_title=traverse_obj(
+                series_details, ('full_prog_details', 0, 'series_title', T(txt_or_none))))

From 21caaf23800c95451cec27dfac56df2c0f8de85a Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 3 Sep 2023 01:13:40 +0100
Subject: [PATCH 705/860] [test] Remove redundancy from lambda expected value
 regex

---
 test/helper.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/test/helper.py b/test/helper.py
index fc55c6b46..5b7e3dfe2 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -142,7 +142,7 @@ def expect_value(self, got, expected, field):
         self.assertTrue(
             contains_str in got,
             'field %s (value: %r) should contain %r' % (field, got, contains_str))
-    elif isinstance(expected, compat_str) and re.match(r'^lambda \w+:', expected):
+    elif isinstance(expected, compat_str) and re.match(r'lambda \w+:', expected):
         fn = eval(expected)
         suite = expected.split(':', 1)[1].strip()
         self.assertTrue(

From bbd3e7e9999877104e1e47a8ed49f3b90257f083 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 3 Sep 2023 01:18:22 +0100
Subject: [PATCH 706/860] [utils] Properly handle list values in update_url()

An actual list value in a query update could have been treated
as a list of values because of the key:list parse_qs format.
---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 81ff78807..fdf41b025 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -4257,7 +4257,7 @@ def update_url(url, **kwargs):
     query = kwargs.pop('query_update', None)
     if query:
         qs = compat_parse_qs(url.query)
-        qs.update(query)
+        qs.update((k, [v]) for k, v in query.items())
         kwargs['query'] = compat_urllib_parse_urlencode(qs, True)
         kwargs = compat_kwargs(kwargs)
     return compat_urllib_parse.urlunparse(url._replace(**kwargs))

From 66ab0814c4baa2dc79c2dd5287bc0ad61a37c5b9 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 3 Sep 2023 23:15:19 +0100
Subject: [PATCH 707/860] [utils] Revert bbd3e7e, updating docstring, test
 instead

---
 test/test_utils.py  | 46 ++++++++++++++++++++++-----------------------
 youtube_dl/utils.py |  3 ++-
 2 files changed, 24 insertions(+), 25 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index fdae1f744..102420fcb 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -62,13 +62,14 @@ from youtube_dl.utils import (
     OnDemandPagedList,
     orderedSet,
     parse_age_limit,
+    parse_bitrate,
     parse_duration,
     parse_filesize,
     parse_codecs,
     parse_count,
     parse_iso8601,
     parse_resolution,
-    parse_bitrate,
+    parse_qs,
     pkcs1pad,
     prepend_extension,
     read_batch_urls,
@@ -125,7 +126,6 @@ from youtube_dl.compat import (
     compat_setenv,
     compat_str,
     compat_urlparse,
-    compat_parse_qs,
 )
 
 
@@ -683,38 +683,36 @@ class TestUtil(unittest.TestCase):
         self.assertTrue(isinstance(data, bytes))
 
     def test_update_url_query(self):
-        def query_dict(url):
-            return compat_parse_qs(compat_urlparse.urlparse(url).query)
-        self.assertEqual(query_dict(update_url_query(
+        self.assertEqual(parse_qs(update_url_query(
             'http://example.com/path', {'quality': ['HD'], 'format': ['mp4']})),
-            query_dict('http://example.com/path?quality=HD&format=mp4'))
-        self.assertEqual(query_dict(update_url_query(
+            parse_qs('http://example.com/path?quality=HD&format=mp4'))
+        self.assertEqual(parse_qs(update_url_query(
             'http://example.com/path', {'system': ['LINUX', 'WINDOWS']})),
-            query_dict('http://example.com/path?system=LINUX&system=WINDOWS'))
-        self.assertEqual(query_dict(update_url_query(
+            parse_qs('http://example.com/path?system=LINUX&system=WINDOWS'))
+        self.assertEqual(parse_qs(update_url_query(
             'http://example.com/path', {'fields': 'id,formats,subtitles'})),
-            query_dict('http://example.com/path?fields=id,formats,subtitles'))
-        self.assertEqual(query_dict(update_url_query(
+            parse_qs('http://example.com/path?fields=id,formats,subtitles'))
+        self.assertEqual(parse_qs(update_url_query(
             'http://example.com/path', {'fields': ('id,formats,subtitles', 'thumbnails')})),
-            query_dict('http://example.com/path?fields=id,formats,subtitles&fields=thumbnails'))
-        self.assertEqual(query_dict(update_url_query(
+            parse_qs('http://example.com/path?fields=id,formats,subtitles&fields=thumbnails'))
+        self.assertEqual(parse_qs(update_url_query(
             'http://example.com/path?manifest=f4m', {'manifest': []})),
-            query_dict('http://example.com/path'))
-        self.assertEqual(query_dict(update_url_query(
+            parse_qs('http://example.com/path'))
+        self.assertEqual(parse_qs(update_url_query(
             'http://example.com/path?system=LINUX&system=WINDOWS', {'system': 'LINUX'})),
-            query_dict('http://example.com/path?system=LINUX'))
-        self.assertEqual(query_dict(update_url_query(
+            parse_qs('http://example.com/path?system=LINUX'))
+        self.assertEqual(parse_qs(update_url_query(
             'http://example.com/path', {'fields': b'id,formats,subtitles'})),
-            query_dict('http://example.com/path?fields=id,formats,subtitles'))
-        self.assertEqual(query_dict(update_url_query(
+            parse_qs('http://example.com/path?fields=id,formats,subtitles'))
+        self.assertEqual(parse_qs(update_url_query(
             'http://example.com/path', {'width': 1080, 'height': 720})),
-            query_dict('http://example.com/path?width=1080&height=720'))
-        self.assertEqual(query_dict(update_url_query(
+            parse_qs('http://example.com/path?width=1080&height=720'))
+        self.assertEqual(parse_qs(update_url_query(
             'http://example.com/path', {'bitrate': 5020.43})),
-            query_dict('http://example.com/path?bitrate=5020.43'))
-        self.assertEqual(query_dict(update_url_query(
+            parse_qs('http://example.com/path?bitrate=5020.43'))
+        self.assertEqual(parse_qs(update_url_query(
             'http://example.com/path', {'test': '第二行тест'})),
-            query_dict('http://example.com/path?test=%E7%AC%AC%E4%BA%8C%E8%A1%8C%D1%82%D0%B5%D1%81%D1%82'))
+            parse_qs('http://example.com/path?test=%E7%AC%AC%E4%BA%8C%E8%A1%8C%D1%82%D0%B5%D1%81%D1%82'))
 
     def test_multipart_encode(self):
         self.assertEqual(
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index fdf41b025..443d2609c 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -4248,6 +4248,7 @@ def update_url(url, **kwargs):
        url: compat_str or parsed URL tuple
        if query_update is in kwargs, update query with
        its value instead of replacing (overrides any `query`)
+       NB: query_update expects parse_qs() format: [key: value_list, ...]
        returns: compat_str
     """
     if not kwargs:
@@ -4257,7 +4258,7 @@ def update_url(url, **kwargs):
     query = kwargs.pop('query_update', None)
     if query:
         qs = compat_parse_qs(url.query)
-        qs.update((k, [v]) for k, v in query.items())
+        qs.update(query)
         kwargs['query'] = compat_urllib_parse_urlencode(qs, True)
         kwargs = compat_kwargs(kwargs)
     return compat_urllib_parse.urlunparse(url._replace(**kwargs))

From 00ef748cc0e35ee60efd0f7a00e373ab8d1af86b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 24 Sep 2023 22:00:13 +0100
Subject: [PATCH 708/860] [downloader] Fix baa6c5e: show ETA of http download
 as ETA instead of total d/l time

---
 youtube_dl/downloader/common.py | 2 +-
 youtube_dl/downloader/http.py   | 2 +-
 2 files changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/common.py b/youtube_dl/downloader/common.py
index afb4ee33d..91e691776 100644
--- a/youtube_dl/downloader/common.py
+++ b/youtube_dl/downloader/common.py
@@ -96,7 +96,7 @@ class FileDownloader(object):
                 return None
             return int(float(remaining) / rate)
         start, now = (start_or_rate, now_or_remaining)
-        total, current = args
+        total, current = args[:2]
         if total is None:
             return None
         if now is None:
diff --git a/youtube_dl/downloader/http.py b/youtube_dl/downloader/http.py
index 28a49b9e8..3cad87420 100644
--- a/youtube_dl/downloader/http.py
+++ b/youtube_dl/downloader/http.py
@@ -294,7 +294,7 @@ class HttpFD(FileDownloader):
 
                 # Progress message
                 speed = self.calc_speed(start, now, byte_counter - ctx.resume_len)
-                eta = self.calc_eta(speed, ctx.data_len and (ctx.data_len - ctx.resume_len))
+                eta = self.calc_eta(speed, ctx.data_len and (ctx.data_len - byte_counter))
 
                 self._hook_progress({
                     'status': 'downloading',

From b7fca0fab36c71fee02d6ecf81acbbaa46942be4 Mon Sep 17 00:00:00 2001
From: ReenigneArcher <42013603+ReenigneArcher@users.noreply.github.com>
Date: Wed, 15 Nov 2023 18:54:31 -0500
Subject: [PATCH 709/860] [Youtube] Update consent cookie handling to match
 site

Apologies for force push!
[skip ci]
---
 youtube_dl/extractor/youtube.py | 14 ++++----------
 1 file changed, 4 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9c419c002..3bf483c1c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -260,16 +260,10 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         cookies = self._get_cookies('https://www.youtube.com/')
         if cookies.get('__Secure-3PSID'):
             return
-        consent_id = None
-        consent = cookies.get('CONSENT')
-        if consent:
-            if 'YES' in consent.value:
-                return
-            consent_id = self._search_regex(
-                r'PENDING\+(\d+)', consent.value, 'consent', default=None)
-        if not consent_id:
-            consent_id = random.randint(100, 999)
-        self._set_cookie('.youtube.com', 'CONSENT', 'YES+cb.20210328-17-p0.en+FX+%s' % consent_id)
+        socs = cookies.get('SOCS')
+        if socs and not socs.value.startswith('CAA'):  # not consented
+            return
+        self._set_cookie('.youtube.com', 'SOCS', 'CAI', secure=True)  # accept all (required for mixes)
 
     def _real_initialize(self):
         self._initialize_consent()

From 4e115e18cbb02ecde30edb736a030cf84bf813e9 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 18 Oct 2023 14:28:10 +0100
Subject: [PATCH 710/860] [workflows/ci.yml] Run apt-get update before
 installing

---
 .github/workflows/ci.yml | 7 +++++++
 1 file changed, 7 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 7fb8f9f83..f00fd0c6b 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -121,6 +121,12 @@ jobs:
           ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download'  || 'nodownload' }}
           run-tests-ext: sh
     steps:
+    - name: Prepare Linux
+      if: ${{ startswith(matrix.os, 'ubuntu') }}
+      shell: bash
+      run: |
+        # apt in runner, if needed, may not be up-to-date
+        sudo apt-get update
     - name: Checkout
       uses: actions/checkout@v3
     #-------- Python 3 -----
@@ -128,6 +134,7 @@ jobs:
       id: setup-python
       if: ${{ matrix.python-impl == 'cpython' && matrix.python-version != '2.6' && matrix.python-version != '2.7' && matrix.python-version != '3.12'}}
       # wrap broken actions/setup-python@v4
+      # NB may run apt-get install in Linux
       uses: ytdl-org/setup-python@v1
       with:
         python-version: ${{ matrix.python-version }}

From 8d227cb97b00a36fa9389bcba2a63ef6db3dbff7 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 28 Nov 2023 16:17:07 +0000
Subject: [PATCH 711/860] [workflows/ci.yml] Actually use default values for
 push and pull_request

---
 .github/workflows/ci.yml | 20 ++++++++++++++++++++
 1 file changed, 20 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index f00fd0c6b..ca52e0e43 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -9,6 +9,7 @@ env:
 
 on:
   push:
+    # push inputs aren't known to GitHub
     inputs:
       cpython-versions:
         type: string
@@ -17,6 +18,7 @@ on:
         type: string
         default: core
   pull_request:
+    # pull_request inputs aren't known to GitHub
     inputs:
       cpython-versions:
         type: string
@@ -56,6 +58,23 @@ jobs:
       test-set: ${{ steps.run.outputs.test-set }}
       own-pip-versions: ${{ steps.run.outputs.own-pip-versions }}
     steps:
+    # push and pull_request inputs aren't known to GitHub (pt3)
+    - name: Set push defaults
+      if: ${{ github.event_name == 'push' }}
+      env:
+        cpython-versions: all
+        test-set: core
+      run: |
+        echo "cpython-versions=${{env.cpython-versions}}" >> "$GITHUB_ENV"
+        echo "test_set=${{env.test_set}}" >> "$GITHUB_ENV"
+    - name: Get pull_request inputs
+      if: ${{ github.event_name == 'pull_request' }}
+      env:
+        cpython-versions: main
+        test-set: both
+      run: |
+        echo "cpython-versions=${{env.cpython-versions}}" >> "$GITHUB_ENV"
+        echo "test_set=${{env.test_set}}" >> "$GITHUB_ENV"
     - name: Make version array
       id: run
       run: |
@@ -79,6 +98,7 @@ jobs:
         # versions with a special get-pip.py in a per-version subdirectory
         printf 'own-pip-versions=%s\n' \
           "$(json_list 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6)" >> "$GITHUB_OUTPUT"
+
   tests:
     name: Run tests
     needs: select

From c6538ed323409707fc73e81fb7c93bc62ad11ac1 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 28 Nov 2023 18:06:40 +0000
Subject: [PATCH 712/860] [workflows/ci.yml] Use setup-python for now released
 Python 3.12

---
 .github/workflows/ci.yml | 34 +++++++++++++++++++---------------
 1 file changed, 19 insertions(+), 15 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index ca52e0e43..93562afd7 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -6,6 +6,9 @@ env:
   pypy-versions: pypy-2.7, pypy-3.6, pypy-3.7
   cpython-versions: main
   test-set: core
+  # Python beta version to be built using pyenv before setup-python support
+  # Must also be included in all-cpython-versions 
+  next: 3.13
 
 on:
   push:
@@ -152,7 +155,7 @@ jobs:
     #-------- Python 3 -----
     - name: Set up supported Python ${{ matrix.python-version }}
       id: setup-python
-      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version != '2.6' && matrix.python-version != '2.7' && matrix.python-version != '3.12'}}
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version != '2.6' && matrix.python-version != '2.7' && matrix.python-version != env.next }}
       # wrap broken actions/setup-python@v4
       # NB may run apt-get install in Linux
       uses: ytdl-org/setup-python@v1
@@ -191,23 +194,23 @@ jobs:
             'import sys' \
             'print(sys.path)' \
             | ${expected} -
-    #-------- Python 3.12 -
-    - name: Set up CPython 3.12 environment
-      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }}
+    #-------- Python next (was 3.12) -
+    - name: Set up CPython 3.next environment
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next }}
       shell: bash
       run: |
         PYENV_ROOT=$HOME/.local/share/pyenv
         echo "PYENV_ROOT=${PYENV_ROOT}" >> "$GITHUB_ENV"
-    - name: Cache Python 3.12
-      id: cache312
-      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }}
+    - name: Cache Python 3.next 
+      id: cachenext
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next }}
       uses: actions/cache@v3
       with:
-        key: python-3.12
+        key: python-${{ env.next }}
         path: |
           ${{ env.PYENV_ROOT }}
-    - name: Build and set up Python 3.12
-      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' && ! steps.cache312.outputs.cache-hit }}
+    - name: Build and set up Python 3.next
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next && ! steps.cachenext.outputs.cache-hit }}
       # dl and build locally
       shell: bash
       run: |
@@ -219,12 +222,13 @@ jobs:
         export PYENV_ROOT=${{ env.PYENV_ROOT }}
         export PATH=$PYENV_ROOT/bin:$PATH
         git clone "https://github.com/pyenv/pyenv.git" "$PYENV_ROOT"
-        pyenv install 3.12.0b4
-    - name: Locate Python 3.12
-      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '3.12' }}
+        pyenv install ${{ env.next }}
+    - name: Locate Python 3.next
+      if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next }}
       shell: bash
       run: |
-        PYTHONHOME="${{ env.PYENV_ROOT }}/versions/3.12.0b4"
+        PYTHONHOME="$(echo "${{ env.PYENV_ROOT }}/versions/${{ env.next }}."*)"
+        test -n "$PYTHONHOME"
         echo "PYTHONHOME=$PYTHONHOME" >> "$GITHUB_ENV"
         echo "PATH=${PYTHONHOME}/bin:$PATH" >> "$GITHUB_ENV"
     #-------- Python 2.7 --
@@ -395,7 +399,7 @@ jobs:
         done
     #-------- nose --------
     - name: Install nose for Python ${{ matrix.python-version }}
-      if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || (matrix.python-impl == 'cpython' && (matrix.python-version == '2.7' || matrix.python-version == '3.12')) }}
+      if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || (matrix.python-impl == 'cpython' && (matrix.python-version == '2.7' || matrix.python-version == env.next)) }}
       shell: bash
       run: |
         echo "$PATH"

From 427472351ce6b2fcf5bb35dde32bf9ee5beddd89 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 28 Nov 2023 17:26:37 +0000
Subject: [PATCH 713/860] [utils] Make restricted filenames ignore characters
 in Unicode categories Mark, Other

Resolves #32629
---
 youtube_dl/utils.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 443d2609c..61b94d84c 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2121,7 +2121,8 @@ def sanitize_filename(s, restricted=False, is_id=False):
         if restricted and (char in '!&\'()[]{}$;`^,#' or char.isspace()):
             return '_'
         if restricted and ord(char) > 127:
-            return '_'
+            return '' if unicodedata.category(char)[0] in 'CM' else '_'
+
         return char
 
     # Replace look-alike Unicode glyphs

From c62936a5f20d941e67d566e74a7c3fc8d8188f7a Mon Sep 17 00:00:00 2001
From: mimvahedi <61986916+mimvahedi@users.noreply.github.com>
Date: Sat, 2 Dec 2023 18:55:09 +0330
Subject: [PATCH 714/860] [telewebion] Fix extraction (#32634)

* [telewebion] fix extraction

Resolves https://github.com/ytdl-org/youtube-dl/issues/5135#issuecomment-932952119

---------

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/telewebion.py | 47 +++++++++++++++---------------
 1 file changed, 23 insertions(+), 24 deletions(-)

diff --git a/youtube_dl/extractor/telewebion.py b/youtube_dl/extractor/telewebion.py
index 1207b1a1b..30192d74e 100644
--- a/youtube_dl/extractor/telewebion.py
+++ b/youtube_dl/extractor/telewebion.py
@@ -3,17 +3,23 @@ from __future__ import unicode_literals
 
 from .common import InfoExtractor
 
+from ..utils import (
+    float_or_none,
+    int_or_none,
+    url_or_none,
+)
+
 
 class TelewebionIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?telewebion\.com/#!/episode/(?P<id>\d+)'
+    _VALID_URL = r'https?://(?:www\.)?telewebion\.com/(episode|clip)/(?P<id>[a-zA-Z0-9]+)'
 
     _TEST = {
-        'url': 'http://www.telewebion.com/#!/episode/1263668/',
+        'url': 'http://www.telewebion.com/episode/0x1b3139c/',
         'info_dict': {
-            'id': '1263668',
+            'id': '0x1b3139c',
             'ext': 'mp4',
             'title': 'قرعه\u200cکشی لیگ قهرمانان اروپا',
-            'thumbnail': r're:^https?://.*\.jpg',
+            'thumbnail': r're:^https?://static\.telewebion\.com/episodeImages/.*/default',
             'view_count': int,
         },
         'params': {
@@ -25,31 +31,24 @@ class TelewebionIE(InfoExtractor):
     def _real_extract(self, url):
         video_id = self._match_id(url)
 
-        secure_token = self._download_webpage(
-            'http://m.s2.telewebion.com/op/op?action=getSecurityToken', video_id)
-        episode_details = self._download_json(
-            'http://m.s2.telewebion.com/op/op', video_id,
-            query={'action': 'getEpisodeDetails', 'episode_id': video_id})
+        episode_details = self._download_json('https://gateway.telewebion.ir/kandoo/episode/getEpisodeDetail/?EpisodeId={0}'.format(video_id), video_id)
+        episode_details = episode_details['body']['queryEpisode'][0]
 
-        m3u8_url = 'http://m.s1.telewebion.com/smil/%s.m3u8?filepath=%s&m3u8=1&secure_token=%s' % (
-            video_id, episode_details['file_path'], secure_token)
+        channel_id = episode_details['channel']['descriptor']
+        episode_image_id = episode_details.get('image')
+        episode_image = 'https://static.telewebion.com/episodeImages/{0}/default'.format(episode_image_id) if episode_image_id else None
+
+        m3u8_url = 'https://cdna.telewebion.com/{0}/episode/{1}/playlist.m3u8'.format(channel_id, video_id)
         formats = self._extract_m3u8_formats(
-            m3u8_url, video_id, ext='mp4', m3u8_id='hls')
-
-        picture_paths = [
-            episode_details.get('picture_path'),
-            episode_details.get('large_picture_path'),
-        ]
-
-        thumbnails = [{
-            'url': picture_path,
-            'preference': idx,
-        } for idx, picture_path in enumerate(picture_paths) if picture_path is not None]
+            m3u8_url, video_id, ext='mp4', m3u8_id='hls',
+            entry_protocol='m3u8_native')
+        self._sort_formats(formats)
 
         return {
             'id': video_id,
             'title': episode_details['title'],
             'formats': formats,
-            'thumbnails': thumbnails,
-            'view_count': episode_details.get('view_count'),
+            'thumbnail': url_or_none(episode_image),
+            'view_count': int_or_none(episode_details.get('view_count')),
+            'duration': float_or_none(episode_details.get('duration')),
         }

From 55a442adaea1eb3dae332fe00179f6dbd437b398 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 5 Dec 2023 20:02:30 +0000
Subject: [PATCH 715/860] [Imgur] Overhaul extractor module (#32612)

Revise extractors for new API and page formats
---
 youtube_dl/extractor/imgur.py | 348 +++++++++++++++++++++++++++-------
 1 file changed, 279 insertions(+), 69 deletions(-)

diff --git a/youtube_dl/extractor/imgur.py b/youtube_dl/extractor/imgur.py
index a5ba03efa..59f129d6a 100644
--- a/youtube_dl/extractor/imgur.py
+++ b/youtube_dl/extractor/imgur.py
@@ -1,101 +1,267 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import re
 
 from .common import InfoExtractor
 from ..utils import (
+    determine_ext,
+    ExtractorError,
+    float_or_none,
     int_or_none,
     js_to_json,
+    merge_dicts,
     mimetype2ext,
-    ExtractorError,
+    parse_iso8601,
+    T,
+    traverse_obj,
+    txt_or_none,
+    url_or_none,
 )
 
 
-class ImgurIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?!(?:a|gallery|(?:t(?:opic)?|r)/[^/]+)/)(?P<id>[a-zA-Z0-9]+)'
+class ImgurBaseIE(InfoExtractor):
+    # hard-coded value, as also used by ArchiveTeam
+    _CLIENT_ID = '546c25a59c58ad7'
+
+    @classmethod
+    def _imgur_result(cls, item_id):
+        return cls.url_result('imgur:%s' % item_id, ImgurIE.ie_key(), item_id)
+
+    def _call_api(self, endpoint, video_id, **kwargs):
+        return self._download_json(
+            'https://api.imgur.com/post/v1/%s/%s?client_id=%s&include=media,account' % (endpoint, video_id, self._CLIENT_ID),
+            video_id, **kwargs)
+
+    @staticmethod
+    def get_description(s):
+        if 'Discover the magic of the internet at Imgur' in s:
+            return None
+        return txt_or_none(s)
+
+
+class ImgurIE(ImgurBaseIE):
+    _VALID_URL = r'''(?x)
+        (?:
+            https?://(?:i\.)?imgur\.com/(?!(?:a|gallery|t|topic|r)/)|
+            imgur:
+        )(?P<id>[a-zA-Z0-9]+)
+    '''
 
     _TESTS = [{
-        'url': 'https://i.imgur.com/A61SaA1.gifv',
+        'url': 'https://imgur.com/A61SaA1',
         'info_dict': {
             'id': 'A61SaA1',
             'ext': 'mp4',
             'title': 're:Imgur GIF$|MRW gifv is up and running without any bugs$',
+            'timestamp': 1416446068,
+            'upload_date': '20141120',
         },
     }, {
-        'url': 'https://imgur.com/A61SaA1',
+        'url': 'https://i.imgur.com/A61SaA1.gifv',
         'only_matching': True,
     }, {
         'url': 'https://i.imgur.com/crGpqCV.mp4',
         'only_matching': True,
     }, {
-        # no title
+        # previously, no title
         'url': 'https://i.imgur.com/jxBXAMC.gifv',
-        'only_matching': True,
+        'info_dict': {
+            'id': 'jxBXAMC',
+            'ext': 'mp4',
+            'title': 'Fahaka puffer feeding',
+            'timestamp': 1533835503,
+            'upload_date': '20180809',
+        },
     }]
 
+    def _extract_twitter_formats(self, html, tw_id='twitter', **kwargs):
+        fatal = kwargs.pop('fatal', False)
+        tw_stream = self._html_search_meta('twitter:player:stream', html, fatal=fatal, **kwargs)
+        if not tw_stream:
+            return []
+        ext = mimetype2ext(self._html_search_meta(
+            'twitter:player:stream:content_type', html, default=None))
+        width, height = (int_or_none(self._html_search_meta('twitter:player:' + v, html, default=None))
+                         for v in ('width', 'height'))
+        return [{
+            'format_id': tw_id,
+            'url': tw_stream,
+            'ext': ext or determine_ext(tw_stream),
+            'width': width,
+            'height': height,
+        }]
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        data = self._call_api('media', video_id, fatal=False, expected_status=404)
         webpage = self._download_webpage(
-            'https://i.imgur.com/{id}.gifv'.format(id=video_id), video_id)
+            'https://i.imgur.com/{id}.gifv'.format(id=video_id), video_id, fatal=not data) or ''
 
-        width = int_or_none(self._og_search_property(
-            'video:width', webpage, default=None))
-        height = int_or_none(self._og_search_property(
-            'video:height', webpage, default=None))
+        if not traverse_obj(data, ('media', 0, (
+                ('type', T(lambda t: t == 'video' or None)),
+                ('metadata', 'is_animated'))), get_all=False):
+            raise ExtractorError(
+                '%s is not a video or animated image' % video_id,
+                expected=True)
+
+        media_fmt = traverse_obj(data, ('media', 0, {
+            'url': ('url', T(url_or_none)),
+            'ext': 'ext',
+            'width': ('width', T(int_or_none)),
+            'height': ('height', T(int_or_none)),
+            'filesize': ('size', T(int_or_none)),
+            'acodec': ('metadata', 'has_sound', T(lambda b: None if b else 'none')),
+        }))
+
+        media_url = traverse_obj(media_fmt, 'url')
+        if media_url:
+            if not media_fmt.get('ext'):
+                media_fmt['ext'] = mimetype2ext(traverse_obj(
+                    data, ('media', 0, 'mime_type'))) or determine_ext(media_url)
+            if traverse_obj(data, ('media', 0, 'type')) == 'image':
+                media_fmt['acodec'] = 'none'
+                media_fmt.setdefault('preference', -10)
+
+        tw_formats = self._extract_twitter_formats(webpage)
+        if traverse_obj(tw_formats, (0, 'url')) == media_url:
+            tw_formats = []
+        else:
+            # maybe this isn't an animated image/video?
+            self._check_formats(tw_formats, video_id)
 
         video_elements = self._search_regex(
             r'(?s)<div class="video-elements">(.*?)</div>',
             webpage, 'video elements', default=None)
-        if not video_elements:
+        if not (video_elements or tw_formats or media_url):
             raise ExtractorError(
-                'No sources found for video %s. Maybe an image?' % video_id,
+                'No sources found for video %s. Maybe a plain image?' % video_id,
                 expected=True)
 
-        formats = []
-        for m in re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements):
-            formats.append({
-                'format_id': m.group('type').partition('/')[2],
-                'url': self._proto_relative_url(m.group('src')),
-                'ext': mimetype2ext(m.group('type')),
-                'width': width,
-                'height': height,
+        def mung_format(fmt, *extra):
+            fmt.update({
                 'http_headers': {
                     'User-Agent': 'youtube-dl (like wget)',
                 },
             })
+            for d in extra:
+                fmt.update(d)
+            return fmt
 
-        gif_json = self._search_regex(
-            r'(?s)var\s+videoItem\s*=\s*(\{.*?\})',
-            webpage, 'GIF code', fatal=False)
-        if gif_json:
-            gifd = self._parse_json(
-                gif_json, video_id, transform_source=js_to_json)
-            formats.append({
-                'format_id': 'gif',
-                'preference': -10,
-                'width': width,
-                'height': height,
-                'ext': 'gif',
-                'acodec': 'none',
-                'vcodec': 'gif',
-                'container': 'gif',
-                'url': self._proto_relative_url(gifd['gifUrl']),
-                'filesize': gifd.get('size'),
-                'http_headers': {
-                    'User-Agent': 'youtube-dl (like wget)',
-                },
-            })
+        if video_elements:
+            def og_get_size(media_type):
+                return dict((p, int_or_none(self._og_search_property(
+                    ':'.join((media_type, p)), webpage, default=None)))
+                    for p in ('width', 'height'))
+
+            size = og_get_size('video')
+            if all(v is None for v in size.values()):
+                size = og_get_size('image')
+
+            formats = traverse_obj(
+                re.finditer(r'<source\s+src="(?P<src>[^"]+)"\s+type="(?P<type>[^"]+)"', video_elements),
+                (Ellipsis, {
+                    'format_id': ('type', T(lambda s: s.partition('/')[2])),
+                    'url': ('src', T(self._proto_relative_url)),
+                    'ext': ('type', T(mimetype2ext)),
+                }, T(lambda f: mung_format(f, size))))
+
+            gif_json = self._search_regex(
+                r'(?s)var\s+videoItem\s*=\s*(\{.*?\})',
+                webpage, 'GIF code', fatal=False)
+            MUST_BRANCH = (None, T(lambda _: None))
+            formats.extend(traverse_obj(gif_json, (
+                T(lambda j: self._parse_json(
+                    j, video_id, transform_source=js_to_json, fatal=False)), {
+                        'url': ('gifUrl', T(self._proto_relative_url)),
+                        'filesize': ('size', T(int_or_none)),
+                }, T(lambda f: mung_format(f, size, {
+                    'format_id': 'gif',
+                    'preference': -10,  # gifs are worse than videos
+                    'ext': 'gif',
+                    'acodec': 'none',
+                    'vcodec': 'gif',
+                    'container': 'gif',
+                })), MUST_BRANCH)))
+        else:
+            formats = []
+
+        # maybe add formats from JSON or page Twitter metadata
+        if not any((u == media_url) for u in traverse_obj(formats, (Ellipsis, 'url'))):
+            formats.append(mung_format(media_fmt))
+        tw_url = traverse_obj(tw_formats, (0, 'url'))
+        if not any((u == tw_url) for u in traverse_obj(formats, (Ellipsis, 'url'))):
+            formats.extend(mung_format(f) for f in tw_formats)
 
         self._sort_formats(formats)
 
-        return {
+        return merge_dicts(traverse_obj(data, {
+            'uploader_id': ('account_id', T(txt_or_none),
+                            T(lambda a: a if int_or_none(a) != 0 else None)),
+            'uploader': ('account', 'username', T(txt_or_none)),
+            'uploader_url': ('account', 'avatar_url', T(url_or_none)),
+            'like_count': ('upvote_count', T(int_or_none)),
+            'dislike_count': ('downvote_count', T(int_or_none)),
+            'comment_count': ('comment_count', T(int_or_none)),
+            'age_limit': ('is_mature', T(lambda x: 18 if x else None)),
+            'timestamp': (('updated_at', 'created_at'), T(parse_iso8601)),
+            'release_timestamp': ('created_at', T(parse_iso8601)),
+        }, get_all=False), traverse_obj(data, ('media', 0, 'metadata', {
+            'title': ('title', T(txt_or_none)),
+            'description': ('description', T(self.get_description)),
+            'duration': ('duration', T(float_or_none)),
+            'timestamp': (('updated_at', 'created_at'), T(parse_iso8601)),
+            'release_timestamp': ('created_at', T(parse_iso8601)),
+        })), {
             'id': video_id,
             'formats': formats,
-            'title': self._og_search_title(webpage, default=video_id),
-        }
+            'title': self._og_search_title(webpage, default='Imgur video ' + video_id),
+            'description': self.get_description(self._og_search_description(webpage)),
+            'thumbnail': url_or_none(self._html_search_meta('thumbnailUrl', webpage, default=None)),
+        })
 
 
-class ImgurGalleryIE(InfoExtractor):
+class ImgurGalleryBaseIE(ImgurBaseIE):
+    _GALLERY = True
+
+    def _real_extract(self, url):
+        gallery_id = self._match_id(url)
+
+        data = self._call_api('albums', gallery_id, fatal=False, expected_status=404)
+
+        info = traverse_obj(data, {
+            'title': ('title', T(txt_or_none)),
+            'description': ('description', T(self.get_description)),
+        })
+
+        if traverse_obj(data, 'is_album'):
+
+            def yield_media_ids():
+                for m_id in traverse_obj(data, (
+                        'media', lambda _, v: v.get('type') == 'video' or v['metadata']['is_animated'],
+                        'id', T(txt_or_none))):
+                    yield m_id
+
+            # if a gallery with exactly one video, apply album metadata to video
+            media_id = (
+                self._GALLERY
+                and traverse_obj(data, ('image_count', T(lambda c: c == 1)))
+                and next(yield_media_ids(), None))
+
+            if not media_id:
+                result = self.playlist_result(
+                    map(self._imgur_result, yield_media_ids()), gallery_id)
+                result.update(info)
+                return result
+            gallery_id = media_id
+
+        result = self._imgur_result(gallery_id)
+        info['_type'] = 'url_transparent'
+        result.update(info)
+        return result
+
+
+class ImgurGalleryIE(ImgurGalleryBaseIE):
     IE_NAME = 'imgur:gallery'
     _VALID_URL = r'https?://(?:i\.)?imgur\.com/(?:gallery|(?:t(?:opic)?|r)/[^/]+)/(?P<id>[a-zA-Z0-9]+)'
 
@@ -106,49 +272,93 @@ class ImgurGalleryIE(InfoExtractor):
             'title': 'Adding faces make every GIF better',
         },
         'playlist_count': 25,
+        'skip': 'Zoinks! You\'ve taken a wrong turn.',
     }, {
+        # TODO: static images - replace with animated/video gallery
         'url': 'http://imgur.com/topic/Aww/ll5Vk',
         'only_matching': True,
     }, {
         'url': 'https://imgur.com/gallery/YcAQlkx',
+        'add_ies': ['Imgur'],
         'info_dict': {
             'id': 'YcAQlkx',
             'ext': 'mp4',
             'title': 'Classic Steve Carell gif...cracks me up everytime....damn the repost downvotes....',
-        }
+            'timestamp': 1358554297,
+            'upload_date': '20130119',
+            'uploader_id': '1648642',
+            'uploader': 'wittyusernamehere',
+        },
     }, {
+        # TODO: static image - replace with animated/video gallery
         'url': 'http://imgur.com/topic/Funny/N8rOudd',
         'only_matching': True,
     }, {
         'url': 'http://imgur.com/r/aww/VQcQPhM',
-        'only_matching': True,
+        'add_ies': ['Imgur'],
+        'info_dict': {
+            'id': 'VQcQPhM',
+            'ext': 'mp4',
+            'title': 'The boss is here',
+            'timestamp': 1476494751,
+            'upload_date': '20161015',
+            'uploader_id': '19138530',
+            'uploader': 'thematrixcam',
+        },
+    },
+        # from PR #16674
+        {
+        'url': 'https://imgur.com/t/unmuted/6lAn9VQ',
+        'info_dict': {
+            'id': '6lAn9VQ',
+            'title': 'Penguins !',
+        },
+        'playlist_count': 3,
+    }, {
+        'url': 'https://imgur.com/t/unmuted/kx2uD3C',
+        'add_ies': ['Imgur'],
+        'info_dict': {
+            'id': 'ZVMv45i',
+            'ext': 'mp4',
+            'title': 'Intruder',
+            'timestamp': 1528129683,
+            'upload_date': '20180604',
+        },
+    }, {
+        'url': 'https://imgur.com/t/unmuted/wXSK0YH',
+        'add_ies': ['Imgur'],
+        'info_dict': {
+            'id': 'JCAP4io',
+            'ext': 'mp4',
+            'title': 're:I got the blues$',
+            'description': 'Luka’s vocal stylings.\n\nFP edit: don’t encourage me. I’ll never stop posting Luka and friends.',
+            'timestamp': 1527809525,
+            'upload_date': '20180531',
+        },
     }]
 
-    def _real_extract(self, url):
-        gallery_id = self._match_id(url)
 
-        data = self._download_json(
-            'https://imgur.com/gallery/%s.json' % gallery_id,
-            gallery_id)['data']['image']
-
-        if data.get('is_album'):
-            entries = [
-                self.url_result('http://imgur.com/%s' % image['hash'], ImgurIE.ie_key(), image['hash'])
-                for image in data['album_images']['images'] if image.get('hash')]
-            return self.playlist_result(entries, gallery_id, data.get('title'), data.get('description'))
-
-        return self.url_result('http://imgur.com/%s' % gallery_id, ImgurIE.ie_key(), gallery_id)
-
-
-class ImgurAlbumIE(ImgurGalleryIE):
+class ImgurAlbumIE(ImgurGalleryBaseIE):
     IE_NAME = 'imgur:album'
     _VALID_URL = r'https?://(?:i\.)?imgur\.com/a/(?P<id>[a-zA-Z0-9]+)'
-
+    _GALLERY = False
     _TESTS = [{
+        # TODO: only static images - replace with animated/video gallery
         'url': 'http://imgur.com/a/j6Orj',
+        'only_matching': True,
+    },
+        # from PR #21693
+        {
+        'url': 'https://imgur.com/a/iX265HX',
         'info_dict': {
-            'id': 'j6Orj',
-            'title': 'A Literary Analysis of "Star Wars: The Force Awakens"',
+            'id': 'iX265HX',
+            'title': 'enen-no-shouboutai'
         },
-        'playlist_count': 12,
+        'playlist_count': 2,
+    }, {
+        'url': 'https://imgur.com/a/8pih2Ed',
+        'info_dict': {
+            'id': '8pih2Ed'
+        },
+        'playlist_mincount': 1,
     }]

From b1bbc1e50277e240419eb1308e444ac8a5da4320 Mon Sep 17 00:00:00 2001
From: Robotix <82544307+realRobotix@users.noreply.github.com>
Date: Wed, 6 Dec 2023 02:17:57 +0100
Subject: [PATCH 716/860] [Epidemic Sound] Add new extractor (#32628)

* Add simple extractor
* Support separate tracks
* Use index as id instead of slug

---------

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/epidemicsound.py | 101 ++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py    |   1 +
 2 files changed, 102 insertions(+)
 create mode 100644 youtube_dl/extractor/epidemicsound.py

diff --git a/youtube_dl/extractor/epidemicsound.py b/youtube_dl/extractor/epidemicsound.py
new file mode 100644
index 000000000..1a52738aa
--- /dev/null
+++ b/youtube_dl/extractor/epidemicsound.py
@@ -0,0 +1,101 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    float_or_none,
+    T,
+    traverse_obj,
+    txt_or_none,
+    unified_timestamp,
+    url_or_none,
+)
+
+
+class EpidemicSoundIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?epidemicsound\.com/track/(?P<id>[0-9a-zA-Z]+)'
+    _TESTS = [{
+        'url': 'https://www.epidemicsound.com/track/yFfQVRpSPz/',
+        'md5': 'd98ff2ddb49e8acab9716541cbc9dfac',
+        'info_dict': {
+            'id': '45014',
+            'display_id': 'yFfQVRpSPz',
+            'ext': 'mp3',
+            'tags': ['foley', 'door', 'knock', 'glass', 'window', 'glass door knock'],
+            'title': 'Door Knock Door 1',
+            'duration': 1,
+            'thumbnail': 'https://cdn.epidemicsound.com/curation-assets/commercial-release-cover-images/default-sfx/3000x3000.jpg',
+            'timestamp': 1415320353,
+            'upload_date': '20141107',
+            'age_limit': None,
+            # check that the "best" format was found, since test file MD5 doesn't
+            # distinguish the formats
+            'format': 'full',
+        },
+    }, {
+        'url': 'https://www.epidemicsound.com/track/mj8GTTwsZd/',
+        'md5': 'c82b745890f9baf18dc2f8d568ee3830',
+        'info_dict': {
+            'id': '148700',
+            'display_id': 'mj8GTTwsZd',
+            'ext': 'mp3',
+            'tags': ['liquid drum n bass', 'energetic'],
+            'title': 'Noplace',
+            'duration': 237,
+            'thumbnail': 'https://cdn.epidemicsound.com/curation-assets/commercial-release-cover-images/11138/3000x3000.jpg',
+            'timestamp': 1694426482,
+            'release_timestamp': 1700535606,
+            'upload_date': '20230911',
+            'age_limit': None,
+            'format': 'full',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        json_data = self._download_json('https://www.epidemicsound.com/json/track/' + video_id, video_id)
+
+        def fmt_or_none(f):
+            if not f.get('format'):
+                f['format'] = f.get('format_id')
+            elif not f.get('format_id'):
+                f['format_id'] = f['format']
+            if not (f['url'] and f['format']):
+                return
+            if f.get('format_note'):
+                f['format_note'] = 'track ID ' + f['format_note']
+            f['preference'] = -1 if f['format'] == 'full' else -2
+            return f
+
+        formats = traverse_obj(json_data, (
+            'stems', T(dict.items), Ellipsis, {
+                'format': (0, T(txt_or_none)),
+                'format_note': (1, 's3TrackId', T(txt_or_none)),
+                'format_id': (1, 'stemType', T(txt_or_none)),
+                'url': (1, 'lqMp3Url', T(url_or_none)),
+            }, T(fmt_or_none)))
+
+        self._sort_formats(formats)
+
+        info = traverse_obj(json_data, {
+            'id': ('id', T(txt_or_none)),
+            'tags': ('metadataTags', Ellipsis, T(txt_or_none)),
+            'title': ('title', T(txt_or_none)),
+            'duration': ('length', T(float_or_none)),
+            'timestamp': ('added', T(unified_timestamp)),
+            'thumbnail': (('imageUrl', 'cover'), T(url_or_none)),
+            'age_limit': ('isExplicit', T(lambda b: 18 if b else None)),
+            'release_timestamp': ('releaseDate', T(unified_timestamp)),
+        }, get_all=False)
+
+        info.update(traverse_obj(json_data, {
+            'categories': ('genres', Ellipsis, 'tag', T(txt_or_none)),
+            'tags': ('metadataTags', Ellipsis, T(txt_or_none)),
+        }))
+
+        info.update({
+            'display_id': video_id,
+            'formats': formats,
+        })
+
+        return info
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index d9289e5bf..82221445f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -357,6 +357,7 @@ from .ellentube import (
 from .elpais import ElPaisIE
 from .embedly import EmbedlyIE
 from .engadget import EngadgetIE
+from .epidemicsound import EpidemicSoundIE
 from .eporner import EpornerIE
 from .eroprofile import EroProfileIE
 from .escapist import EscapistIE

From be008e657d79832642e2158557c899249c9e31cd Mon Sep 17 00:00:00 2001
From: mk-pmb <mk-pmb@users.noreply.github.com>
Date: Wed, 13 Sep 2023 20:57:05 +0200
Subject: [PATCH 717/860] [core] Fix format string injection for metadata JSON
 filename message.

---
 youtube_dl/YoutubeDL.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 13a41928f..6f2aba5ac 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -2635,12 +2635,12 @@ class YoutubeDL(object):
             self.to_screen(msg('[info] %s is already present', label.title()))
             return 'exists'
         else:
-            self.to_screen(msg('[info] Writing %s as JSON to: ' + infofn, label))
+            self.to_screen(msg('[info] Writing %s as JSON to: ', label) + infofn)
             try:
                 write_json_file(self.filter_requested_info(info_dict), infofn)
                 return True
             except (OSError, IOError):
-                self.report_error(msg('Cannot write %s to JSON file ' + infofn, label))
+                self.report_error(msg('Cannot write %s to JSON file ', label) + infofn)
                 return
 
     def _write_thumbnails(self, info_dict, filename):

From 66518714169185195a359e173cef73fba31d76b8 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 20 Jan 2024 18:28:52 +0000
Subject: [PATCH 718/860] [compat] Rework compat for `method` parameter of
 `compat_urllib_request.Request` constructor * fixes #32573 * does not break
 `utils.HEADrequest` (eg)

---
 test/test_compat.py  | 14 ++++++++++++++
 youtube_dl/compat.py | 27 +++++++++++++++++----------
 2 files changed, 31 insertions(+), 10 deletions(-)

diff --git a/test/test_compat.py b/test/test_compat.py
index e233b1ae1..b83c8cb41 100644
--- a/test/test_compat.py
+++ b/test/test_compat.py
@@ -23,6 +23,7 @@ from youtube_dl.compat import (
     compat_urllib_parse_unquote,
     compat_urllib_parse_unquote_plus,
     compat_urllib_parse_urlencode,
+    compat_urllib_request,
 )
 
 
@@ -135,6 +136,19 @@ class TestCompat(unittest.TestCase):
         self.assertEqual(compat_casefold('\u03a3'), '\u03c3')
         self.assertEqual(compat_casefold('A\u0345\u03a3'), 'a\u03b9\u03c3')
 
+    def test_compat_urllib_request_Request(self):
+        self.assertEqual(
+            compat_urllib_request.Request('http://127.0.0.1', method='PUT').get_method(),
+            'PUT')
+
+        class PUTrequest(compat_urllib_request.Request):
+            def get_method(self):
+                return 'PUT'
+
+        self.assertEqual(
+            PUTrequest('http://127.0.0.1').get_method(),
+            'PUT')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 3c526a78d..818ccebd0 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -58,19 +58,26 @@ except ImportError:  # Python 2
 
 # Also fix up lack of method arg in old Pythons
 try:
-    _req = compat_urllib_request.Request
-    _req('http://127.0.0.1', method='GET')
+    type(compat_urllib_request.Request('http://127.0.0.1', method='GET'))
 except TypeError:
-    class _request(object):
-        def __new__(cls, url, *args, **kwargs):
-            method = kwargs.pop('method', None)
-            r = _req(url, *args, **kwargs)
-            if method:
-                r.get_method = types.MethodType(lambda _: method, r)
-            return r
+    def _add_init_method_arg(cls):
 
-    compat_urllib_request.Request = _request
+        init = cls.__init__
 
+        def wrapped_init(self, *args, **kwargs):
+            method = kwargs.pop('method', 'GET')
+            init(self, *args, **kwargs)
+            if any(callable(x.__dict__.get('get_method')) for x in (self.__class__, self) if x != cls):
+                # allow instance or its subclass to override get_method()
+                return
+            if self.has_data() and method == 'GET':
+                method = 'POST'
+            self.get_method = types.MethodType(lambda _: method, self)
+
+        cls.__init__ = wrapped_init
+
+    _add_init_method_arg(compat_urllib_request.Request)
+    del _add_init_method_arg
 
 try:
     import urllib.error as compat_urllib_error

From 640d39f03ae80a0b8d0605a711d97c10f6edbd3f Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 15 Jan 2024 18:32:06 +0000
Subject: [PATCH 719/860] [InfoExtractor] Support some warning and
 `._downloader` shortcut methods from yt-dlp

---
 youtube_dl/extractor/common.py | 56 ++++++++++++++++++++++++++++++++--
 1 file changed, 53 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 0eca9f844..d33557135 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -596,6 +596,14 @@ class InfoExtractor(object):
         """Sets the downloader for this IE."""
         self._downloader = downloader
 
+    @property
+    def cache(self):
+        return self._downloader.cache
+
+    @property
+    def cookiejar(self):
+        return self._downloader.cookiejar
+
     def _real_initialize(self):
         """Real initialization process. Redefine in subclasses."""
         pass
@@ -942,14 +950,47 @@ class InfoExtractor(object):
             else:
                 self.report_warning(errmsg + str(ve))
 
-    def report_warning(self, msg, video_id=None):
+    def __ie_msg(self, *msg):
+        return '[{0}] {1}'.format(self.IE_NAME, ''.join(msg))
+
+    # msg, video_id=None, *args, only_once=False, **kwargs
+    def report_warning(self, msg, *args, **kwargs):
+        if len(args) > 0:
+            video_id = args[0]
+            args = args[1:]
+        else:
+            video_id = kwargs.pop('video_id', None)
         idstr = '' if video_id is None else '%s: ' % video_id
         self._downloader.report_warning(
-            '[%s] %s%s' % (self.IE_NAME, idstr, msg))
+            self.__ie_msg(idstr, msg), *args, **kwargs)
 
     def to_screen(self, msg):
         """Print msg to screen, prefixing it with '[ie_name]'"""
-        self._downloader.to_screen('[%s] %s' % (self.IE_NAME, msg))
+        self._downloader.to_screen(self.__ie_msg(msg))
+
+    def write_debug(self, msg, only_once=False, _cache=[]):
+        '''Log debug message or Print message to stderr'''
+        if not self.get_param('verbose', False):
+            return
+        message = '[debug] ' + self.__ie_msg(msg)
+        logger = self.get_param('logger')
+        if logger:
+            logger.debug(message)
+        else:
+            if only_once and hash(message) in _cache:
+                return
+            self._downloader.to_stderr(message)
+            _cache.append(hash(message))
+
+    # name, default=None, *args, **kwargs
+    def get_param(self, name, *args, **kwargs):
+        default, args = (args[0], args[1:]) if len(args) > 0 else (kwargs.pop('default', None), args)
+        if self._downloader:
+            return self._downloader.params.get(name, default, *args, **kwargs)
+        return default
+
+    def report_drm(self, video_id):
+        self.raise_no_formats('This video is DRM protected', expected=True, video_id=video_id)
 
     def report_extraction(self, id_or_name):
         """Report information extraction."""
@@ -977,6 +1018,15 @@ class InfoExtractor(object):
     def raise_geo_restricted(msg='This video is not available from your location due to geo restriction', countries=None):
         raise GeoRestrictedError(msg, countries=countries)
 
+    def raise_no_formats(self, msg, expected=False, video_id=None):
+        if expected and (
+                self.get_param('ignore_no_formats_error') or self.get_param('wait_for_video')):
+            self.report_warning(msg, video_id)
+        elif isinstance(msg, ExtractorError):
+            raise msg
+        else:
+            raise ExtractorError(msg, expected=expected, video_id=video_id)
+
     # Methods for following #608
     @staticmethod
     def url_result(url, ie=None, video_id=None, video_title=None):

From f8b0135850f39609f72002f5426883859579fc51 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 15 Jan 2024 18:34:21 +0000
Subject: [PATCH 720/860] [YouTube] Rework n-sig processing, realigning with
 yt-dlp * apply n-sig before chunked fragments, fixes #32692

---
 youtube_dl/extractor/youtube.py | 488 +++++++++++++++++++-------------
 1 file changed, 296 insertions(+), 192 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 3bf483c1c..cd4b3ef60 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2,6 +2,7 @@
 
 from __future__ import unicode_literals
 
+import collections
 import itertools
 import json
 import os.path
@@ -23,10 +24,10 @@ from ..compat import (
 )
 from ..jsinterp import JSInterpreter
 from ..utils import (
-    ExtractorError,
     clean_html,
     dict_get,
     error_to_compat_str,
+    ExtractorError,
     float_or_none,
     extract_attributes,
     get_element_by_attribute,
@@ -36,6 +37,7 @@ from ..utils import (
     LazyList,
     merge_dicts,
     mimetype2ext,
+    NO_DEFAULT,
     parse_codecs,
     parse_duration,
     parse_qs,
@@ -45,6 +47,7 @@ from ..utils import (
     str_or_none,
     str_to_int,
     traverse_obj,
+    try_call,
     try_get,
     txt_or_none,
     unescapeHTML,
@@ -1460,6 +1463,30 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         self._code_cache = {}
         self._player_cache = {}
 
+    # *ytcfgs, webpage=None
+    def _extract_player_url(self, *ytcfgs, **kw_webpage):
+        if ytcfgs and not isinstance(ytcfgs[0], dict):
+            webpage = kw_webpage.get('webpage') or ytcfgs[0]
+        if webpage:
+            player_url = self._search_regex(
+                r'"(?:PLAYER_JS_URL|jsUrl)"\s*:\s*"([^"]+)"',
+                webpage or '', 'player URL', fatal=False)
+            if player_url:
+                ytcfgs = ytcfgs + ({'PLAYER_JS_URL': player_url},)
+        return traverse_obj(
+            ytcfgs, (Ellipsis, 'PLAYER_JS_URL'), (Ellipsis, 'WEB_PLAYER_CONTEXT_CONFIGS', Ellipsis, 'jsUrl'),
+            get_all=False, expected_type=lambda u: urljoin('https://www.youtube.com', u))
+
+    def _download_player_url(self, video_id, fatal=False):
+        res = self._download_webpage(
+            'https://www.youtube.com/iframe_api',
+            note='Downloading iframe API JS', video_id=video_id, fatal=fatal)
+        player_version = self._search_regex(
+            r'player\\?/([0-9a-fA-F]{8})\\?/', res or '', 'player version', fatal=fatal,
+            default=NO_DEFAULT if res else None)
+        if player_version:
+            return 'https://www.youtube.com/s/player/{0}/player_ias.vflset/en_US/base.js'.format(player_version)
+
     def _signature_cache_id(self, example_sig):
         """ Return a string representation of a signature """
         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
@@ -1474,46 +1501,49 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             raise ExtractorError('Cannot identify player %r' % player_url)
         return id_m.group('id')
 
-    def _get_player_code(self, video_id, player_url, player_id=None):
+    def _load_player(self, video_id, player_url, fatal=True, player_id=None):
         if not player_id:
             player_id = self._extract_player_info(player_url)
-
         if player_id not in self._code_cache:
-            self._code_cache[player_id] = self._download_webpage(
-                player_url, video_id,
+            code = self._download_webpage(
+                player_url, video_id, fatal=fatal,
                 note='Downloading player ' + player_id,
                 errnote='Download of %s failed' % player_url)
-        return self._code_cache[player_id]
+            if code:
+                self._code_cache[player_id] = code
+        return self._code_cache[player_id] if fatal else self._code_cache.get(player_id)
 
     def _extract_signature_function(self, video_id, player_url, example_sig):
         player_id = self._extract_player_info(player_url)
 
         # Read from filesystem cache
-        func_id = 'js_%s_%s' % (
+        func_id = 'js_{0}_{1}'.format(
             player_id, self._signature_cache_id(example_sig))
         assert os.path.basename(func_id) == func_id
 
-        cache_spec = self._downloader.cache.load('youtube-sigfuncs', func_id)
-        if cache_spec is not None:
-            return lambda s: ''.join(s[i] for i in cache_spec)
+        self.write_debug('Extracting signature function {0}'.format(func_id))
+        cache_spec, code = self.cache.load('youtube-sigfuncs', func_id), None
 
-        code = self._get_player_code(video_id, player_url, player_id)
-        res = self._parse_sig_js(code)
+        if not cache_spec:
+            code = self._load_player(video_id, player_url, player_id)
+        if code:
+            res = self._parse_sig_js(code)
+            test_string = ''.join(map(compat_chr, range(len(example_sig))))
+            cache_spec = [ord(c) for c in res(test_string)]
+            self.cache.store('youtube-sigfuncs', func_id, cache_spec)
 
-        test_string = ''.join(map(compat_chr, range(len(example_sig))))
-        cache_res = res(test_string)
-        cache_spec = [ord(c) for c in cache_res]
-
-        self._downloader.cache.store('youtube-sigfuncs', func_id, cache_spec)
-        return res
+        return lambda s: ''.join(s[i] for i in cache_spec)
 
     def _print_sig_code(self, func, example_sig):
+        if not self.get_param('youtube_print_sig_code'):
+            return
+
         def gen_sig_code(idxs):
             def _genslice(start, end, step):
                 starts = '' if start == 0 else str(start)
                 ends = (':%d' % (end + step)) if end + step >= 0 else ':'
                 steps = '' if step == 1 else (':%d' % step)
-                return 's[%s%s%s]' % (starts, ends, steps)
+                return 's[{0}{1}{2}]'.format(starts, ends, steps)
 
             step = None
             # Quelch pyflakes warnings - start will be set when step is set
@@ -1564,143 +1594,137 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             jscode, 'Initial JS player signature function name', group='sig')
 
         jsi = JSInterpreter(jscode)
-
         initial_function = jsi.extract_function(funcname)
-
         return lambda s: initial_function([s])
 
+    def _cached(self, func, *cache_id):
+        def inner(*args, **kwargs):
+            if cache_id not in self._player_cache:
+                try:
+                    self._player_cache[cache_id] = func(*args, **kwargs)
+                except ExtractorError as e:
+                    self._player_cache[cache_id] = e
+                except Exception as e:
+                    self._player_cache[cache_id] = ExtractorError(traceback.format_exc(), cause=e)
+
+            ret = self._player_cache[cache_id]
+            if isinstance(ret, Exception):
+                raise ret
+            return ret
+        return inner
+
     def _decrypt_signature(self, s, video_id, player_url):
         """Turn the encrypted s field into a working signature"""
-
-        if player_url is None:
-            raise ExtractorError('Cannot decrypt signature without player_url')
-
-        try:
-            player_id = (player_url, self._signature_cache_id(s))
-            if player_id not in self._player_cache:
-                func = self._extract_signature_function(
-                    video_id, player_url, s
-                )
-                self._player_cache[player_id] = func
-            func = self._player_cache[player_id]
-            if self._downloader.params.get('youtube_print_sig_code'):
-                self._print_sig_code(func, s)
-            return func(s)
-        except Exception as e:
-            tb = traceback.format_exc()
-            raise ExtractorError(
-                'Signature extraction failed: ' + tb, cause=e)
-
-    def _extract_player_url(self, webpage):
-        player_url = self._search_regex(
-            r'"(?:PLAYER_JS_URL|jsUrl)"\s*:\s*"([^"]+)"',
-            webpage or '', 'player URL', fatal=False)
-        if not player_url:
-            return
-        if player_url.startswith('//'):
-            player_url = 'https:' + player_url
-        elif not re.match(r'https?://', player_url):
-            player_url = compat_urllib_parse.urljoin(
-                'https://www.youtube.com', player_url)
-        return player_url
+        extract_sig = self._cached(
+            self._extract_signature_function, 'sig', player_url, self._signature_cache_id(s))
+        func = extract_sig(video_id, player_url, s)
+        self._print_sig_code(func, s)
+        return func(s)
 
     # from yt-dlp
     # See also:
     # 1. https://github.com/ytdl-org/youtube-dl/issues/29326#issuecomment-894619419
     # 2. https://code.videolan.org/videolan/vlc/-/blob/4fb284e5af69aa9ac2100ccbdd3b88debec9987f/share/lua/playlist/youtube.lua#L116
     # 3. https://github.com/ytdl-org/youtube-dl/issues/30097#issuecomment-950157377
-    def _extract_n_function_name(self, jscode):
-        target = r'(?P<nfunc>[a-zA-Z_$][\w$]*)(?:\[(?P<idx>\d+)\])?'
-        nfunc_and_idx = self._search_regex(
-            r'\.get\("n"\)\)&&\(b=(%s)\([\w$]+\)' % (target, ),
-            jscode, 'Initial JS player n function name')
-        nfunc, idx = re.match(target, nfunc_and_idx).group('nfunc', 'idx')
-        if not idx:
-            return nfunc
-
-        VAR_RE_TMPL = r'var\s+%s\s*=\s*(?P<name>\[(?P<alias>%s)\])[;,]'
-        note = 'Initial JS player n function {0} (%s[%s])' % (nfunc, idx)
-
-        def search_function_code(needle, group):
-            return self._search_regex(
-                VAR_RE_TMPL % (re.escape(nfunc), needle), jscode,
-                note.format(group), group=group)
-
-        if int_or_none(idx) == 0:
-            real_nfunc = search_function_code(r'[a-zA-Z_$][\w$]*', group='alias')
-            if real_nfunc:
-                return real_nfunc
-        return self._parse_json(
-            search_function_code('.+?', group='name'),
-            nfunc, transform_source=js_to_json)[int(idx)]
-
-    def _extract_n_function(self, video_id, player_url):
-        player_id = self._extract_player_info(player_url)
-        func_code = self._downloader.cache.load('youtube-nsig', player_id)
-
-        if func_code:
-            jsi = JSInterpreter(func_code)
-        else:
-            jscode = self._get_player_code(video_id, player_url, player_id)
-            funcname = self._extract_n_function_name(jscode)
-            jsi = JSInterpreter(jscode)
-            func_code = jsi.extract_function_code(funcname)
-            self._downloader.cache.store('youtube-nsig', player_id, func_code)
-
-        if self._downloader.params.get('youtube_print_sig_code'):
-            self.to_screen('Extracted nsig function from {0}:\n{1}\n'.format(player_id, func_code[1]))
-
-        return lambda s: jsi.extract_function_from_code(*func_code)([s])
-
-    def _n_descramble(self, n_param, player_url, video_id):
-        """Compute the response to YT's "n" parameter challenge,
-           or None
-
-        Args:
-        n_param     -- challenge string that is the value of the
-                       URL's "n" query parameter
-        player_url  -- URL of YT player JS
-        video_id
-        """
-
-        sig_id = ('nsig_value', n_param)
-        if sig_id in self._player_cache:
-            return self._player_cache[sig_id]
+    def _decrypt_nsig(self, n, video_id, player_url):
+        """Turn the encrypted n field into a working signature"""
+        if player_url is None:
+            raise ExtractorError('Cannot decrypt nsig without player_url')
 
         try:
-            player_id = ('nsig', player_url)
-            if player_id not in self._player_cache:
-                self._player_cache[player_id] = self._extract_n_function(video_id, player_url)
-            func = self._player_cache[player_id]
-            ret = func(n_param)
-            if ret.startswith('enhanced_except_'):
-                raise ExtractorError('Unhandled exception in decode')
-            self._player_cache[sig_id] = ret
-            if self._downloader.params.get('verbose', False):
-                self._downloader.to_screen('[debug] [%s] %s' % (self.IE_NAME, 'Decrypted nsig {0} => {1}'.format(n_param, self._player_cache[sig_id])))
-            return self._player_cache[sig_id]
-        except Exception as e:
-            self._downloader.report_warning(
-                '[%s] %s (%s %s)' % (
-                    self.IE_NAME,
-                    'Unable to decode n-parameter: download likely to be throttled',
+            jsi, player_id, func_code = self._extract_n_function_code(video_id, player_url)
+        except ExtractorError as e:
+            raise ExtractorError('Unable to extract nsig jsi, player_id, func_codefunction code', cause=e)
+        if self.get_param('youtube_print_sig_code'):
+            self.to_screen('Extracted nsig function from {0}:\n{1}\n'.format(
+                player_id, func_code[1]))
+
+        try:
+            extract_nsig = self._cached(self._extract_n_function_from_code, 'nsig func', player_url)
+            ret = extract_nsig(jsi, func_code)(n)
+        except JSInterpreter.Exception as e:
+            self.report_warning(
+                '%s (%s %s)' % (
+                    self.__ie_msg(
+                        'Unable to decode n-parameter: download likely to be throttled'),
                     error_to_compat_str(e),
                     traceback.format_exc()))
+            return
+
+        self.write_debug('Decrypted nsig {0} => {1}'.format(n, ret))
+        return ret
+
+    def _extract_n_function_name(self, jscode):
+        func_name, idx = self._search_regex(
+            r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z_$][\w$]*)(?:\[(?P<idx>\d+)\])?\([\w$]+\)',
+            jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
+        if not idx:
+            return func_name
+
+        return self._parse_json(self._search_regex(
+            r'var {0}\s*=\s*(\[.+?\])\s*[,;]'.format(re.escape(func_name)), jscode,
+            'Initial JS player n function list ({0}.{1})'.format(func_name, idx)),
+            func_name, transform_source=js_to_json)[int(idx)]
+
+    def _extract_n_function_code(self, video_id, player_url):
+        player_id = self._extract_player_info(player_url)
+        func_code = self.cache.load('youtube-nsig', player_id)
+        jscode = func_code or self._load_player(video_id, player_url)
+        jsi = JSInterpreter(jscode)
+
+        if func_code:
+            return jsi, player_id, func_code
+
+        func_name = self._extract_n_function_name(jscode)
+
+        # For redundancy
+        func_code = self._search_regex(
+            r'''(?xs)%s\s*=\s*function\s*\((?P<var>[\w$]+)\)\s*
+                     # NB: The end of the regex is intentionally kept strict
+                     {(?P<code>.+?}\s*return\ [\w$]+.join\(""\))};''' % func_name,
+            jscode, 'nsig function', group=('var', 'code'), default=None)
+        if func_code:
+            func_code = ([func_code[0]], func_code[1])
+        else:
+            self.write_debug('Extracting nsig function with jsinterp')
+            func_code = jsi.extract_function_code(func_name)
+
+        self.cache.store('youtube-nsig', player_id, func_code)
+        return jsi, player_id, func_code
+
+    def _extract_n_function_from_code(self, jsi, func_code):
+        func = jsi.extract_function_from_code(*func_code)
+
+        def extract_nsig(s):
+            try:
+                ret = func([s])
+            except JSInterpreter.Exception:
+                raise
+            except Exception as e:
+                raise JSInterpreter.Exception(traceback.format_exc(), cause=e)
+
+            if ret.startswith('enhanced_except_'):
+                raise JSInterpreter.Exception('Signature function returned an exception')
+            return ret
+
+        return extract_nsig
+
+    def _unthrottle_format_urls(self, video_id, player_url, *formats):
+
+        def decrypt_nsig(n):
+            return self._cached(self._decrypt_nsig, 'nsig', n, player_url)
 
-    def _unthrottle_format_urls(self, video_id, player_url, formats):
         for fmt in formats:
             parsed_fmt_url = compat_urllib_parse.urlparse(fmt['url'])
             n_param = compat_parse_qs(parsed_fmt_url.query).get('n')
             if not n_param:
                 continue
             n_param = n_param[-1]
-            n_response = self._n_descramble(n_param, player_url, video_id)
+            n_response = decrypt_nsig(n_param)(n_param, video_id, player_url)
             if n_response is None:
                 # give up if descrambling failed
                 break
-            for fmt_dct in traverse_obj(fmt, (None, (None, ('fragments', Ellipsis))), expected_type=dict):
-                fmt_dct['url'] = update_url(
-                    fmt_dct['url'], query_update={'n': [n_response]})
+            fmt['url'] = update_url_query(fmt['url'], {'n': n_response})
 
     # from yt-dlp, with tweaks
     def _extract_signature_timestamp(self, video_id, player_url, ytcfg=None, fatal=False):
@@ -1708,16 +1732,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         Extract signatureTimestamp (sts)
         Required to tell API what sig/player version is in use.
         """
-        sts = int_or_none(ytcfg.get('STS')) if isinstance(ytcfg, dict) else None
+        sts = traverse_obj(ytcfg, 'STS', expected_type=int)
         if not sts:
             # Attempt to extract from player
             if player_url is None:
                 error_msg = 'Cannot extract signature timestamp without player_url.'
                 if fatal:
                     raise ExtractorError(error_msg)
-                self._downloader.report_warning(error_msg)
+                self.report_warning(error_msg)
                 return
-            code = self._get_player_code(video_id, player_url)
+            code = self._load_player(video_id, player_url, fatal=fatal)
             sts = int_or_none(self._search_regex(
                 r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code or '',
                 'JS player signature timestamp', group='sts', fatal=fatal))
@@ -1733,12 +1757,18 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         # cpn generation algorithm is reverse engineered from base.js.
         # In fact it works even with dummy cpn.
         CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
-        cpn = ''.join((CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16)))
+        cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
 
-        playback_url = update_url(
-            playback_url, query_update={
-                'ver': ['2'],
-                'cpn': [cpn],
+        # more consistent results setting it to right before the end
+        qs = parse_qs(playback_url)
+        video_length = '{0}'.format(float((qs.get('len') or ['1.5'])[0]) - 1)
+
+        playback_url = update_url_query(
+            playback_url, {
+                'ver': '2',
+                'cpn': cpn,
+                'cmt': video_length,
+                'el': 'detailpage',  # otherwise defaults to "shorts"
             })
 
         self._download_webpage(
@@ -1986,8 +2016,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             else:
                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)
 
+        if not player_url:
+            player_url = self._extract_player_url(webpage)
+
         formats = []
-        itags = []
+        itags = collections.defaultdict(set)
         itag_qualities = {}
         q = qualities(['tiny', 'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'])
         CHUNK_SIZE = 10 << 20
@@ -2003,58 +2036,92 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 })
             } for range_start in range(0, f['filesize'], CHUNK_SIZE))
 
+        lower = lambda s: s.lower()
+
         for fmt in streaming_formats:
-            if fmt.get('targetDurationSec') or fmt.get('drmFamilies'):
+            if fmt.get('targetDurationSec'):
                 continue
 
             itag = str_or_none(fmt.get('itag'))
-            quality = fmt.get('quality')
-            if itag and quality:
+            audio_track = traverse_obj(fmt, ('audioTrack', T(dict))) or {}
+
+            quality = traverse_obj(fmt, ((
+                # The 3gp format (17) in android client has a quality of "small",
+                # but is actually worse than other formats
+                T(lambda _: 'tiny' if itag == 17 else None),
+                ('quality', T(lambda q: q if q and q != 'tiny' else None)),
+                ('audioQuality', T(lower)),
+                'quality'), T(txt_or_none)), get_all=False)
+            if quality and itag:
                 itag_qualities[itag] = quality
             # FORMAT_STREAM_TYPE_OTF(otf=1) requires downloading the init fragment
             # (adding `&sq=0` to the URL) and parsing emsg box to determine the
-            # number of fragment that would subsequently requested with (`&sq=N`)
+            # number of fragments that would subsequently be requested with (`&sq=N`)
             if fmt.get('type') == 'FORMAT_STREAM_TYPE_OTF':
                 continue
 
             fmt_url = fmt.get('url')
             if not fmt_url:
                 sc = compat_parse_qs(fmt.get('signatureCipher'))
-                fmt_url = url_or_none(try_get(sc, lambda x: x['url'][0]))
-                encrypted_sig = try_get(sc, lambda x: x['s'][0])
-                if not (sc and fmt_url and encrypted_sig):
+                fmt_url = traverse_obj(sc, ('url', -1, T(url_or_none)))
+                encrypted_sig = traverse_obj(sc, ('s', -1))
+                if not (fmt_url and encrypted_sig):
                     continue
-                if not player_url:
-                    player_url = self._extract_player_url(webpage)
+                player_url = player_url or self._extract_player_url(webpage)
                 if not player_url:
                     continue
-                signature = self._decrypt_signature(sc['s'][0], video_id, player_url)
-                sp = try_get(sc, lambda x: x['sp'][0]) or 'signature'
-                fmt_url += '&' + sp + '=' + signature
+                try:
+                    fmt_url = update_url_query(fmt_url, {
+                        traverse_obj(sc, ('sp', -1)) or 'signature':
+                            [self._decrypt_signature(encrypted_sig, video_id, player_url)],
+                    })
+                except ExtractorError as e:
+                    self.report_warning('Signature extraction failed: Some formats may be missing',
+                                        video_id=video_id, only_once=True)
+                    self.write_debug(error_to_compat_str(e), only_once=True)
+                    continue
 
-            if itag:
-                itags.append(itag)
-            tbr = float_or_none(
-                fmt.get('averageBitrate') or fmt.get('bitrate'), 1000)
+            language_preference = (
+                10 if audio_track.get('audioIsDefault')
+                else -10 if 'descriptive' in (traverse_obj(audio_track, ('displayName', T(lower))) or '')
+                else -1)
+            name = (
+                traverse_obj(fmt, ('qualityLabel', T(txt_or_none)))
+                or quality.replace('audio_quality_', ''))
             dct = {
-                'asr': int_or_none(fmt.get('audioSampleRate')),
-                'filesize': int_or_none(fmt.get('contentLength')),
-                'format_id': itag,
-                'format_note': fmt.get('qualityLabel') or quality,
-                'fps': int_or_none(fmt.get('fps')),
-                'height': int_or_none(fmt.get('height')),
-                'quality': q(quality),
-                'tbr': tbr,
+                'format_id': join_nonempty(itag, fmt.get('isDrc') and 'drc'),
                 'url': fmt_url,
-                'width': fmt.get('width'),
+                # Format 22 is likely to be damaged: see https://github.com/yt-dlp/yt-dlp/issues/3372
+                'source_preference': ((-5 if itag == '22' else -1)
+                                      + (100 if 'Premium' in name else 0)),
+                'quality': q(quality),
+                'language': join_nonempty(audio_track.get('id', '').split('.')[0],
+                                          'desc' if language_preference < -1 else '') or None,
+                'language_preference': language_preference,
+                # Strictly de-prioritize 3gp formats
+                'preference': -2 if itag == '17' else None,
             }
-            mimetype = fmt.get('mimeType')
-            if mimetype:
-                mobj = re.match(
-                    r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', mimetype)
-                if mobj:
-                    dct['ext'] = mimetype2ext(mobj.group(1))
-                    dct.update(parse_codecs(mobj.group(2)))
+            if itag:
+                itags[itag].add(('https', dct.get('language')))
+            self._unthrottle_format_urls(video_id, player_url, dct)
+            dct.update(traverse_obj(fmt, {
+                'asr': ('audioSampleRate', T(int_or_none)),
+                'filesize': ('contentLength', T(int_or_none)),
+                'format_note': ('qualityLabel', T(lambda x: x or quality)),
+                # for some formats, fps is wrongly returned as 1
+                'fps': ('fps', T(int_or_none), T(lambda f: f if f > 1 else None)),
+                'audio_channels': ('audioChannels', T(int_or_none)),
+                'height': ('height', T(int_or_none)),
+                'has_drm': ('drmFamilies', T(bool)),
+                'tbr': (('averageBitrate', 'bitrate'), T(lambda t: float_or_none(t, 1000))),
+                'width': ('width', T(int_or_none)),
+                '_duration_ms': ('approxDurationMs', T(int_or_none)),
+            }, get_all=False))
+            mime_mobj = re.match(
+                r'((?:[^/]+)/(?:[^;]+))(?:;\s*codecs="([^"]+)")?', fmt.get('mimeType') or '')
+            if mime_mobj:
+                dct['ext'] = mimetype2ext(mime_mobj.group(1))
+                dct.update(parse_codecs(mime_mobj.group(2)))
             single_stream = 'none' in (dct.get(c) for c in ('acodec', 'vcodec'))
             if single_stream and dct.get('ext'):
                 dct['container'] = dct['ext'] + '_dash'
@@ -2069,32 +2136,62 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
             formats.append(dct)
 
+        def process_manifest_format(f, proto, client_name, itag, all_formats=False):
+            key = (proto, f.get('language'))
+            if not all_formats and key in itags[itag]:
+                return False
+            itags[itag].add(key)
+
+            if itag:
+                f['format_id'] = (
+                    '{0}-{1}'.format(itag, proto)
+                    if all_formats or any(p != proto for p, _ in itags[itag])
+                    else itag)
+
+            if f.get('source_preference') is None:
+                f['source_preference'] = -1
+
+            if itag in ('616', '235'):
+                f['format_note'] = join_nonempty(f.get('format_note'), 'Premium', delim=' ')
+                f['source_preference'] += 100
+
+            f['quality'] = q(traverse_obj(f, (
+                'format_id', T(lambda s: itag_qualities[s.split('-')[0]])), default=-1))
+            if try_call(lambda: f['fps'] <= 1):
+                del f['fps']
+
+            if proto == 'hls' and f.get('has_drm'):
+                f['has_drm'] = 'maybe'
+                f['source_preference'] -= 5
+            return True
+
         hls_manifest_url = streaming_data.get('hlsManifestUrl')
         if hls_manifest_url:
             for f in self._extract_m3u8_formats(
                     hls_manifest_url, video_id, 'mp4', fatal=False):
-                itag = self._search_regex(
-                    r'/itag/(\d+)', f['url'], 'itag', default=None)
-                if itag:
-                    f['format_id'] = itag
-                formats.append(f)
+                if process_manifest_format(
+                        f, 'hls', None, self._search_regex(
+                            r'/itag/(\d+)', f['url'], 'itag', default=None)):
+                    formats.append(f)
 
         if self._downloader.params.get('youtube_include_dash_manifest', True):
             dash_manifest_url = streaming_data.get('dashManifestUrl')
             if dash_manifest_url:
                 for f in self._extract_mpd_formats(
                         dash_manifest_url, video_id, fatal=False):
-                    itag = f['format_id']
-                    if itag in itags:
-                        continue
-                    if itag in itag_qualities:
-                        f['quality'] = q(itag_qualities[itag])
-                    filesize = int_or_none(self._search_regex(
-                        r'/clen/(\d+)', f.get('fragment_base_url')
-                        or f['url'], 'file size', default=None))
-                    if filesize:
-                        f['filesize'] = filesize
-                    formats.append(f)
+                    if process_manifest_format(
+                            f, 'dash', None, f['format_id']):
+                        f['filesize'] = traverse_obj(f, (
+                            ('fragment_base_url', 'url'), T(lambda u: self._search_regex(
+                                r'/clen/(\d+)', u, 'file size', default=None)),
+                            T(int_or_none)), get_all=False)
+                        formats.append(f)
+
+        playable_formats = [f for f in formats if not f.get('has_drm')]
+        if formats and not playable_formats:
+            # If there are no formats that definitely don't have DRM, all have DRM
+            self.report_drm(video_id)
+        formats[:] = playable_formats
 
         if not formats:
             if streaming_data.get('licenseInfos'):
@@ -2166,6 +2263,17 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             video_details.get('lengthSeconds')
             or microformat.get('lengthSeconds')) \
             or parse_duration(search_meta('duration'))
+
+        for f in formats:
+            # Some formats may have much smaller duration than others (possibly damaged during encoding)
+            # but avoid false positives with small duration differences.
+            # Ref: https://github.com/yt-dlp/yt-dlp/issues/2823
+            if try_call(lambda x: float(x.pop('_duration_ms')) / duration < 500, args=(f,)):
+                self.report_warning(
+                    '{0}: Some possibly damaged formats will be deprioritized'.format(video_id), only_once=True)
+                # Strictly de-prioritize damaged formats
+                f['preference'] = -10
+
         is_live = video_details.get('isLive')
 
         owner_profile_url = self._yt_urljoin(self._extract_author_var(
@@ -2174,10 +2282,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         uploader = self._extract_author_var(
             webpage, 'name', videodetails=video_details, metadata=microformat)
 
-        if not player_url:
-            player_url = self._extract_player_url(webpage)
-        self._unthrottle_format_urls(video_id, player_url, formats)
-
         info = {
             'id': video_id,
             'title': self._live_title(video_title) if is_live else video_title,

From dc512e3a8a26a8e3fc7f1f67e5ee5e7699db8659 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 15 Jan 2024 18:38:43 +0000
Subject: [PATCH 721/860] [YouTube] Fix `like_count` extraction using
 `likeButtonViewModel` * also fix various tests * TODO: check against yt-dlp
 tests

---
 test/test_subtitles.py          |  1 +
 youtube_dl/extractor/youtube.py | 18 ++++++++++++++----
 2 files changed, 15 insertions(+), 4 deletions(-)

diff --git a/test/test_subtitles.py b/test/test_subtitles.py
index 1197721ff..e005c78fc 100644
--- a/test/test_subtitles.py
+++ b/test/test_subtitles.py
@@ -295,6 +295,7 @@ class TestNRKSubtitles(BaseTestSubtitles):
     def test_allsubtitles(self):
         self.DL.params['writesubtitles'] = True
         self.DL.params['allsubtitles'] = True
+        self.DL.params['format'] = 'best/bestvideo'
         subtitles = self.getSubtitles()
         self.assertEqual(set(subtitles.keys()), set(['nb-ttv']))
         self.assertEqual(md5(subtitles['nb-ttv']), '67e06ff02d0deaf975e68f6cb8f6a149')
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index cd4b3ef60..db840fc45 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -39,6 +39,7 @@ from ..utils import (
     mimetype2ext,
     NO_DEFAULT,
     parse_codecs,
+    parse_count,
     parse_duration,
     parse_qs,
     qualities,
@@ -46,6 +47,7 @@ from ..utils import (
     smuggle_url,
     str_or_none,
     str_to_int,
+    T,
     traverse_obj,
     try_call,
     try_get,
@@ -1250,7 +1252,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'title': 'IMG 3456',
                 'description': '',
                 'upload_date': '20170613',
-                'uploader': 'ElevageOrVert',
+                'uploader': "l'Or Vert asbl",
                 'uploader_id': '@ElevageOrVert',
             },
             'params': {
@@ -2474,6 +2476,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             'like_count': str_to_int(like_count),
                             'dislike_count': str_to_int(dislike_count),
                         })
+                    else:
+                        info['like_count'] = traverse_obj(vpir, (
+                            'videoActions', 'menuRenderer', 'topLevelButtons', Ellipsis,
+                            'segmentedLikeDislikeButtonViewModel', 'likeButtonViewModel', 'likeButtonViewModel',
+                            'toggleButtonViewModel', 'toggleButtonViewModel', 'defaultButtonViewModel',
+                            'buttonViewModel', (('title', ('accessibilityText', T(lambda s: s.split()), Ellipsis))), T(parse_count)),
+                            get_all=False)
+
                 vsir = content.get('videoSecondaryInfoRenderer')
                 if vsir:
                     rows = try_get(
@@ -2588,7 +2598,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
         'playlist_mincount': 94,
         'info_dict': {
             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
-            'title': 'Igor Kleiner - Playlists',
+            'title': r're:Igor Kleiner(?: Ph\.D\.)? - Playlists',
             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
             'uploader': 'Igor Kleiner',
             'uploader_id': '@IgorDataScience',
@@ -2599,7 +2609,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
         'playlist_mincount': 94,
         'info_dict': {
             'id': 'UCqj7Cz7revf5maW9g5pgNcg',
-            'title': 'Igor Kleiner - Playlists',
+            'title': r're:Igor Kleiner(?: Ph\.D\.)? - Playlists',
             'description': 'md5:be97ee0f14ee314f1f002cf187166ee2',
             'uploader': 'Igor Kleiner',
             'uploader_id': '@IgorDataScience',
@@ -2711,7 +2721,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
         'url': 'https://www.youtube.com/channel/UCKfVa3S1e4PHvxWcwyMMg8w/channels',
         'info_dict': {
             'id': 'UCKfVa3S1e4PHvxWcwyMMg8w',
-            'title': 'lex will - Channels',
+            'title': r're:lex will - (?:Home|Channels)',
             'description': 'md5:2163c5d0ff54ed5f598d6a7e6211e488',
             'uploader': 'lex will',
             'uploader_id': '@lexwill718',

From c58b655a9ef255eb9d02b4d57706c46cfdf35975 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 27 Jan 2024 15:29:25 +0000
Subject: [PATCH 722/860] [InfoExtractor] Support DASH subtitle extraction
 (yt-dlp back-port)

---
 test/test_InfoExtractor.py     | 113 +++++++++++-
 youtube_dl/extractor/common.py | 312 ++++++++++++++++++++-------------
 2 files changed, 300 insertions(+), 125 deletions(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 3f96645de..043b62243 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -993,7 +993,8 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'tbr': 5997.485,
                     'width': 1920,
                     'height': 1080,
-                }]
+                }],
+                {},
             ), (
                 # https://github.com/ytdl-org/youtube-dl/pull/14844
                 'urls_only',
@@ -1076,7 +1077,8 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'tbr': 4400,
                     'width': 1920,
                     'height': 1080,
-                }]
+                }],
+                {},
             ), (
                 # https://github.com/ytdl-org/youtube-dl/issues/20346
                 # Media considered unfragmented even though it contains
@@ -1122,18 +1124,119 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'width': 360,
                     'height': 360,
                     'fps': 30,
-                }]
+                }],
+                {},
+            ), (
+                'subtitles',
+                'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
+                'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/',
+                [{
+                    'format_id': 'audio=128001',
+                    'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
+                    'ext': 'm4a',
+                    'tbr': 128.001,
+                    'asr': 48000,
+                    'format_note': 'DASH audio',
+                    'container': 'm4a_dash',
+                    'vcodec': 'none',
+                    'acodec': 'mp4a.40.2',
+                    'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
+                    'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/',
+                    'protocol': 'http_dash_segments',
+                }, {
+                    'format_id': 'video=100000',
+                    'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
+                    'ext': 'mp4',
+                    'width': 336,
+                    'height': 144,
+                    'tbr': 100,
+                    'format_note': 'DASH video',
+                    'container': 'mp4_dash',
+                    'vcodec': 'avc1.4D401F',
+                    'acodec': 'none',
+                    'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
+                    'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/',
+                    'protocol': 'http_dash_segments',
+                }, {
+                    'format_id': 'video=326000',
+                    'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
+                    'ext': 'mp4',
+                    'width': 562,
+                    'height': 240,
+                    'tbr': 326,
+                    'format_note': 'DASH video',
+                    'container': 'mp4_dash',
+                    'vcodec': 'avc1.4D401F',
+                    'acodec': 'none',
+                    'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
+                    'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/',
+                    'protocol': 'http_dash_segments',
+                }, {
+                    'format_id': 'video=698000',
+                    'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
+                    'ext': 'mp4',
+                    'width': 844,
+                    'height': 360,
+                    'tbr': 698,
+                    'format_note': 'DASH video',
+                    'container': 'mp4_dash',
+                    'vcodec': 'avc1.4D401F',
+                    'acodec': 'none',
+                    'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
+                    'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/',
+                    'protocol': 'http_dash_segments',
+                }, {
+                    'format_id': 'video=1493000',
+                    'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
+                    'ext': 'mp4',
+                    'width': 1126,
+                    'height': 480,
+                    'tbr': 1493,
+                    'format_note': 'DASH video',
+                    'container': 'mp4_dash',
+                    'vcodec': 'avc1.4D401F',
+                    'acodec': 'none',
+                    'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
+                    'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/',
+                    'protocol': 'http_dash_segments',
+                }, {
+                    'format_id': 'video=4482000',
+                    'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
+                    'ext': 'mp4',
+                    'width': 1688,
+                    'height': 720,
+                    'tbr': 4482,
+                    'format_note': 'DASH video',
+                    'container': 'mp4_dash',
+                    'vcodec': 'avc1.4D401F',
+                    'acodec': 'none',
+                    'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
+                    'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/',
+                    'protocol': 'http_dash_segments',
+                }],
+                {
+                    'en': [
+                        {
+                            'ext': 'mp4',
+                            'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
+                            'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
+                            'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/',
+                            'protocol': 'http_dash_segments',
+                        }
+                    ]
+                },
             )
         ]
 
-        for mpd_file, mpd_url, mpd_base_url, expected_formats in _TEST_CASES:
+        for mpd_file, mpd_url, mpd_base_url, expected_formats, expected_subtitles in _TEST_CASES:
             with open('./test/testdata/mpd/%s.mpd' % mpd_file,
                       mode='r', encoding='utf-8') as f:
-                formats = self.ie._parse_mpd_formats(
+                formats, subtitles = self.ie._parse_mpd_formats_and_subtitles(
                     compat_etree_fromstring(f.read().encode('utf-8')),
                     mpd_base_url=mpd_base_url, mpd_url=mpd_url)
                 self.ie._sort_formats(formats)
                 expect_value(self, formats, expected_formats, None)
+                expect_value(self, subtitles, expected_subtitles, None)
 
     def test_parse_f4m_formats(self):
         _TEST_CASES = [
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index d33557135..ed55d3e07 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2,6 +2,7 @@
 from __future__ import unicode_literals
 
 import base64
+import collections
 import datetime
 import functools
 import hashlib
@@ -58,6 +59,7 @@ from ..utils import (
     GeoRestrictedError,
     GeoUtils,
     int_or_none,
+    join_nonempty,
     js_to_json,
     JSON_LD_RE,
     mimetype2ext,
@@ -74,6 +76,7 @@ from ..utils import (
     str_or_none,
     str_to_int,
     strip_or_none,
+    T,
     traverse_obj,
     try_get,
     unescapeHTML,
@@ -1751,6 +1754,12 @@ class InfoExtractor(object):
             'format_note': 'Quality selection URL',
         }
 
+    def _report_ignoring_subs(self, name):
+        self.report_warning(bug_reports_message(
+            'Ignoring subtitle tracks found in the {0} manifest; '
+            'if any subtitle tracks are missing,'.format(name)
+        ), only_once=True)
+
     def _extract_m3u8_formats(self, m3u8_url, video_id, ext=None,
                               entry_protocol='m3u8', preference=None,
                               m3u8_id=None, note=None, errnote=None,
@@ -2191,23 +2200,46 @@ class InfoExtractor(object):
             })
         return entries
 
-    def _extract_mpd_formats(self, mpd_url, video_id, mpd_id=None, note=None, errnote=None, fatal=True, data=None, headers={}, query={}):
+    def _extract_mpd_formats(self, *args, **kwargs):
+        fmts, subs = self._extract_mpd_formats_and_subtitles(*args, **kwargs)
+        if subs:
+            self._report_ignoring_subs('DASH')
+        return fmts
+
+    def _extract_mpd_formats_and_subtitles(
+            self, mpd_url, video_id, mpd_id=None, note=None, errnote=None,
+            fatal=True, data=None, headers=None, query=None):
+
+        # TODO: or not? param not yet implemented
+        if self.get_param('ignore_no_formats_error'):
+            fatal = False
+
         res = self._download_xml_handle(
             mpd_url, video_id,
-            note=note or 'Downloading MPD manifest',
-            errnote=errnote or 'Failed to download MPD manifest',
-            fatal=fatal, data=data, headers=headers, query=query)
+            note='Downloading MPD manifest' if note is None else note,
+            errnote='Failed to download MPD manifest' if errnote is None else errnote,
+            fatal=fatal, data=data, headers=headers or {}, query=query or {})
         if res is False:
-            return []
+            return [], {}
         mpd_doc, urlh = res
         if mpd_doc is None:
-            return []
-        mpd_base_url = base_url(urlh.geturl())
+            return [], {}
 
-        return self._parse_mpd_formats(
+        # We could have been redirected to a new url when we retrieved our mpd file.
+        mpd_url = urlh.geturl()
+        mpd_base_url = base_url(mpd_url)
+
+        return self._parse_mpd_formats_and_subtitles(
             mpd_doc, mpd_id, mpd_base_url, mpd_url)
 
-    def _parse_mpd_formats(self, mpd_doc, mpd_id=None, mpd_base_url='', mpd_url=None):
+    def _parse_mpd_formats(self, *args, **kwargs):
+        fmts, subs = self._parse_mpd_formats_and_subtitles(*args, **kwargs)
+        if subs:
+            self._report_ignoring_subs('DASH')
+        return fmts
+
+    def _parse_mpd_formats_and_subtitles(
+            self, mpd_doc, mpd_id=None, mpd_base_url='', mpd_url=None):
         """
         Parse formats from MPD manifest.
         References:
@@ -2215,8 +2247,10 @@ class InfoExtractor(object):
             http://standards.iso.org/ittf/PubliclyAvailableStandards/c065274_ISO_IEC_23009-1_2014.zip
          2. https://en.wikipedia.org/wiki/Dynamic_Adaptive_Streaming_over_HTTP
         """
-        if mpd_doc.get('type') == 'dynamic':
-            return []
+        # TODO: param not yet implemented: default like previous yt-dl logic
+        if not self.get_param('dynamic_mpd', False):
+            if mpd_doc.get('type') == 'dynamic':
+                return [], {}
 
         namespace = self._search_regex(r'(?i)^{([^}]+)?}MPD$', mpd_doc.tag, 'namespace', default=None)
 
@@ -2228,6 +2262,7 @@ class InfoExtractor(object):
 
         def extract_multisegment_info(element, ms_parent_info):
             ms_info = ms_parent_info.copy()
+            base_url = ms_info.get('base_url')
 
             # As per [1, 5.3.9.2.2] SegmentList and SegmentTemplate share some
             # common attributes and elements.  We will only extract relevant
@@ -2285,7 +2320,8 @@ class InfoExtractor(object):
             return ms_info
 
         mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration'))
-        formats = []
+        formats, subtitles = [], {}
+        stream_numbers = collections.defaultdict(int)
         for period in mpd_doc.findall(_add_ns('Period')):
             period_duration = parse_duration(period.get('duration')) or mpd_duration
             period_ms_info = extract_multisegment_info(period, {
@@ -2295,7 +2331,7 @@ class InfoExtractor(object):
             for adaptation_set in period.findall(_add_ns('AdaptationSet')):
                 if is_drm_protected(adaptation_set):
                     continue
-                adaption_set_ms_info = extract_multisegment_info(adaptation_set, period_ms_info)
+                adaptation_set_ms_info = extract_multisegment_info(adaptation_set, period_ms_info)
                 for representation in adaptation_set.findall(_add_ns('Representation')):
                     if is_drm_protected(representation):
                         continue
@@ -2303,27 +2339,35 @@ class InfoExtractor(object):
                     representation_attrib.update(representation.attrib)
                     # According to [1, 5.3.7.2, Table 9, page 41], @mimeType is mandatory
                     mime_type = representation_attrib['mimeType']
-                    content_type = mime_type.split('/')[0]
-                    if content_type == 'text':
-                        # TODO implement WebVTT downloading
-                        pass
-                    elif content_type in ('video', 'audio'):
-                        base_url = ''
-                        for element in (representation, adaptation_set, period, mpd_doc):
-                            base_url_e = element.find(_add_ns('BaseURL'))
-                            if base_url_e is not None:
-                                base_url = base_url_e.text + base_url
-                                if re.match(r'^https?://', base_url):
-                                    break
-                        if mpd_base_url and not re.match(r'^https?://', base_url):
-                            if not mpd_base_url.endswith('/') and not base_url.startswith('/'):
-                                mpd_base_url += '/'
-                            base_url = mpd_base_url + base_url
-                        representation_id = representation_attrib.get('id')
-                        lang = representation_attrib.get('lang')
-                        url_el = representation.find(_add_ns('BaseURL'))
-                        filesize = int_or_none(url_el.attrib.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el is not None else None)
-                        bandwidth = int_or_none(representation_attrib.get('bandwidth'))
+                    content_type = representation_attrib.get('contentType') or mime_type.split('/')[0]
+                    codec_str = representation_attrib.get('codecs', '')
+                    # Some kind of binary subtitle found in some youtube livestreams
+                    if mime_type == 'application/x-rawcc':
+                        codecs = {'scodec': codec_str}
+                    else:
+                        codecs = parse_codecs(codec_str)
+                    if content_type not in ('video', 'audio', 'text'):
+                        if mime_type == 'image/jpeg':
+                            content_type = mime_type
+                        elif codecs.get('vcodec', 'none') != 'none':
+                            content_type = 'video'
+                        elif codecs.get('acodec', 'none') != 'none':
+                            content_type = 'audio'
+                        elif codecs.get('scodec', 'none') != 'none':
+                            content_type = 'text'
+                        elif mimetype2ext(mime_type) in ('tt', 'dfxp', 'ttml', 'xml', 'json'):
+                            content_type = 'text'
+                        else:
+                            self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
+                            continue
+
+                    representation_id = representation_attrib.get('id')
+                    lang = representation_attrib.get('lang')
+                    url_el = representation.find(_add_ns('BaseURL'))
+                    filesize = int_or_none(url_el.get('{http://youtube.com/yt/2012/10/10}contentLength') if url_el is not None else None)
+                    bandwidth = int_or_none(representation_attrib.get('bandwidth'))
+                    format_id = join_nonempty(representation_id or content_type, mpd_id)
+                    if content_type in ('video', 'audio'):
                         f = {
                             'format_id': '%s-%s' % (mpd_id, representation_id) if mpd_id else representation_id,
                             'manifest_url': mpd_url,
@@ -2338,104 +2382,125 @@ class InfoExtractor(object):
                             'filesize': filesize,
                             'container': mimetype2ext(mime_type) + '_dash',
                         }
-                        f.update(parse_codecs(representation_attrib.get('codecs')))
-                        representation_ms_info = extract_multisegment_info(representation, adaption_set_ms_info)
+                        f.update(codecs)
+                    elif content_type == 'text':
+                        f = {
+                            'ext': mimetype2ext(mime_type),
+                            'manifest_url': mpd_url,
+                            'filesize': filesize,
+                        }
+                    elif content_type == 'image/jpeg':
+                        # See test case in VikiIE
+                        # https://www.viki.com/videos/1175236v-choosing-spouse-by-lottery-episode-1
+                        f = {
+                            'format_id': format_id,
+                            'ext': 'mhtml',
+                            'manifest_url': mpd_url,
+                            'format_note': 'DASH storyboards (jpeg)',
+                            'acodec': 'none',
+                            'vcodec': 'none',
+                        }
+                    if is_drm_protected(adaptation_set) or is_drm_protected(representation):
+                        f['has_drm'] = True
+                    representation_ms_info = extract_multisegment_info(representation, adaptation_set_ms_info)
 
-                        def prepare_template(template_name, identifiers):
-                            tmpl = representation_ms_info[template_name]
-                            # First of, % characters outside $...$ templates
-                            # must be escaped by doubling for proper processing
-                            # by % operator string formatting used further (see
-                            # https://github.com/ytdl-org/youtube-dl/issues/16867).
-                            t = ''
-                            in_template = False
-                            for c in tmpl:
+                    def prepare_template(template_name, identifiers):
+                        tmpl = representation_ms_info[template_name]
+                        # First of, % characters outside $...$ templates
+                        # must be escaped by doubling for proper processing
+                        # by % operator string formatting used further (see
+                        # https://github.com/ytdl-org/youtube-dl/issues/16867).
+                        t = ''
+                        in_template = False
+                        for c in tmpl:
+                            t += c
+                            if c == '$':
+                                in_template = not in_template
+                            elif c == '%' and not in_template:
                                 t += c
-                                if c == '$':
-                                    in_template = not in_template
-                                elif c == '%' and not in_template:
-                                    t += c
-                            # Next, $...$ templates are translated to their
-                            # %(...) counterparts to be used with % operator
-                            t = t.replace('$RepresentationID$', representation_id)
-                            t = re.sub(r'\$(%s)\$' % '|'.join(identifiers), r'%(\1)d', t)
-                            t = re.sub(r'\$(%s)%%([^$]+)\$' % '|'.join(identifiers), r'%(\1)\2', t)
-                            t.replace('$$', '$')
-                            return t
+                        # Next, $...$ templates are translated to their
+                        # %(...) counterparts to be used with % operator
+                        t = t.replace('$RepresentationID$', representation_id)
+                        t = re.sub(r'\$(%s)\$' % '|'.join(identifiers), r'%(\1)d', t)
+                        t = re.sub(r'\$(%s)%%([^$]+)\$' % '|'.join(identifiers), r'%(\1)\2', t)
+                        t.replace('$$', '$')
+                        return t
 
-                        # @initialization is a regular template like @media one
-                        # so it should be handled just the same way (see
-                        # https://github.com/ytdl-org/youtube-dl/issues/11605)
-                        if 'initialization' in representation_ms_info:
-                            initialization_template = prepare_template(
-                                'initialization',
-                                # As per [1, 5.3.9.4.2, Table 15, page 54] $Number$ and
-                                # $Time$ shall not be included for @initialization thus
-                                # only $Bandwidth$ remains
-                                ('Bandwidth', ))
-                            representation_ms_info['initialization_url'] = initialization_template % {
-                                'Bandwidth': bandwidth,
-                            }
+                    # @initialization is a regular template like @media one
+                    # so it should be handled just the same way (see
+                    # https://github.com/ytdl-org/youtube-dl/issues/11605)
+                    if 'initialization' in representation_ms_info:
+                        initialization_template = prepare_template(
+                            'initialization',
+                            # As per [1, 5.3.9.4.2, Table 15, page 54] $Number$ and
+                            # $Time$ shall not be included for @initialization thus
+                            # only $Bandwidth$ remains
+                            ('Bandwidth', ))
+                        representation_ms_info['initialization_url'] = initialization_template % {
+                            'Bandwidth': bandwidth,
+                        }
 
-                        def location_key(location):
-                            return 'url' if re.match(r'^https?://', location) else 'path'
+                    def location_key(location):
+                        return 'url' if re.match(r'^https?://', location) else 'path'
 
-                        if 'segment_urls' not in representation_ms_info and 'media' in representation_ms_info:
+                    if 'segment_urls' not in representation_ms_info and 'media' in representation_ms_info:
 
-                            media_template = prepare_template('media', ('Number', 'Bandwidth', 'Time'))
-                            media_location_key = location_key(media_template)
+                        media_template = prepare_template('media', ('Number', 'Bandwidth', 'Time'))
+                        media_location_key = location_key(media_template)
 
-                            # As per [1, 5.3.9.4.4, Table 16, page 55] $Number$ and $Time$
-                            # can't be used at the same time
-                            if '%(Number' in media_template and 's' not in representation_ms_info:
-                                segment_duration = None
-                                if 'total_number' not in representation_ms_info and 'segment_duration' in representation_ms_info:
-                                    segment_duration = float_or_none(representation_ms_info['segment_duration'], representation_ms_info['timescale'])
-                                    representation_ms_info['total_number'] = int(math.ceil(float(period_duration) / segment_duration))
-                                representation_ms_info['fragments'] = [{
-                                    media_location_key: media_template % {
-                                        'Number': segment_number,
-                                        'Bandwidth': bandwidth,
-                                    },
-                                    'duration': segment_duration,
-                                } for segment_number in range(
-                                    representation_ms_info['start_number'],
-                                    representation_ms_info['total_number'] + representation_ms_info['start_number'])]
-                            else:
-                                # $Number*$ or $Time$ in media template with S list available
-                                # Example $Number*$: http://www.svtplay.se/klipp/9023742/stopptid-om-bjorn-borg
-                                # Example $Time$: https://play.arkena.com/embed/avp/v2/player/media/b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe/1/129411
-                                representation_ms_info['fragments'] = []
-                                segment_time = 0
-                                segment_d = None
-                                segment_number = representation_ms_info['start_number']
+                        # As per [1, 5.3.9.4.4, Table 16, page 55] $Number$ and $Time$
+                        # can't be used at the same time
+                        if '%(Number' in media_template and 's' not in representation_ms_info:
+                            segment_duration = None
+                            if 'total_number' not in representation_ms_info and 'segment_duration' in representation_ms_info:
+                                segment_duration = float_or_none(representation_ms_info['segment_duration'], representation_ms_info['timescale'])
+                                representation_ms_info['total_number'] = int(math.ceil(
+                                    float_or_none(period_duration, segment_duration, default=0)))
+                            representation_ms_info['fragments'] = [{
+                                media_location_key: media_template % {
+                                    'Number': segment_number,
+                                    'Bandwidth': bandwidth,
+                                },
+                                'duration': segment_duration,
+                            } for segment_number in range(
+                                representation_ms_info['start_number'],
+                                representation_ms_info['total_number'] + representation_ms_info['start_number'])]
+                        else:
+                            # $Number*$ or $Time$ in media template with S list available
+                            # Example $Number*$: http://www.svtplay.se/klipp/9023742/stopptid-om-bjorn-borg
+                            # Example $Time$: https://play.arkena.com/embed/avp/v2/player/media/b41dda37-d8e7-4d3f-b1b5-9a9db578bdfe/1/129411
+                            representation_ms_info['fragments'] = []
+                            segment_time = 0
+                            segment_d = None
+                            segment_number = representation_ms_info['start_number']
 
-                                def add_segment_url():
-                                    segment_url = media_template % {
-                                        'Time': segment_time,
-                                        'Bandwidth': bandwidth,
-                                        'Number': segment_number,
-                                    }
-                                    representation_ms_info['fragments'].append({
-                                        media_location_key: segment_url,
-                                        'duration': float_or_none(segment_d, representation_ms_info['timescale']),
-                                    })
+                            def add_segment_url():
+                                segment_url = media_template % {
+                                    'Time': segment_time,
+                                    'Bandwidth': bandwidth,
+                                    'Number': segment_number,
+                                }
+                                representation_ms_info['fragments'].append({
+                                    media_location_key: segment_url,
+                                    'duration': float_or_none(segment_d, representation_ms_info['timescale']),
+                                })
 
-                                for num, s in enumerate(representation_ms_info['s']):
-                                    segment_time = s.get('t') or segment_time
-                                    segment_d = s['d']
+                            for num, s in enumerate(representation_ms_info['s']):
+                                segment_time = s.get('t') or segment_time
+                                segment_d = s['d']
+                                add_segment_url()
+                                segment_number += 1
+                                for r in range(s.get('r', 0)):
+                                    segment_time += segment_d
                                     add_segment_url()
                                     segment_number += 1
-                                    for r in range(s.get('r', 0)):
-                                        segment_time += segment_d
-                                        add_segment_url()
-                                        segment_number += 1
-                                    segment_time += segment_d
-                        elif 'segment_urls' in representation_ms_info and 's' in representation_ms_info:
+                                segment_time += segment_d
+                    elif 'segment_urls' in representation_ms_info:
+                        fragments = []
+                        if 's' in representation_ms_info:
                             # No media template
                             # Example: https://www.youtube.com/watch?v=iXZV5uAYMJI
                             # or any YouTube dashsegments video
-                            fragments = []
                             segment_index = 0
                             timescale = representation_ms_info['timescale']
                             for s in representation_ms_info['s']:
@@ -2487,8 +2552,15 @@ class InfoExtractor(object):
                             f['url'] = base_url
                         formats.append(f)
                     else:
-                        self.report_warning('Unknown MIME type %s in DASH manifest' % mime_type)
-        return formats
+                        # Assuming direct URL to unfragmented media.
+                        f['url'] = representation_ms_info['base_url']
+                    if content_type in ('video', 'audio', 'image/jpeg'):
+                        f['manifest_stream_number'] = stream_numbers[f['url']]
+                        stream_numbers[f['url']] += 1
+                        formats.append(f)
+                    elif content_type == 'text':
+                        subtitles.setdefault(lang or 'und', []).append(f)
+        return formats, subtitles
 
     def _extract_ism_formats(self, ism_url, video_id, ism_id=None, note=None, errnote=None, fatal=True, data=None, headers={}, query={}):
         res = self._download_xml_handle(

From bec9180e8904a12c55cfa838b0541879d16bf20f Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 27 Jan 2024 00:07:14 +0000
Subject: [PATCH 723/860] [downloader/dash] Support `range` in fragment (format
 f'{start}-{end}')  * adapted from
 https://github.com/ytdl-org/youtube-dl/pull/30279  * thx former GH user
 kikuyan

---
 youtube_dl/downloader/dash.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/downloader/dash.py b/youtube_dl/downloader/dash.py
index 2800d4260..f3c058879 100644
--- a/youtube_dl/downloader/dash.py
+++ b/youtube_dl/downloader/dash.py
@@ -35,6 +35,7 @@ class DashSegmentsFD(FragmentFD):
         for frag_index, fragment in enumerate(fragments, 1):
             if frag_index <= ctx['fragment_index']:
                 continue
+            success = False
             # In DASH, the first segment contains necessary headers to
             # generate a valid MP4 file, so always abort for the first segment
             fatal = frag_index == 1 or not skip_unavailable_fragments
@@ -42,10 +43,14 @@ class DashSegmentsFD(FragmentFD):
             if not fragment_url:
                 assert fragment_base_url
                 fragment_url = urljoin(fragment_base_url, fragment['path'])
-            success = False
+            headers = info_dict.get('http_headers')
+            fragment_range = fragment.get('range')
+            if fragment_range:
+                headers = headers.copy() if headers else {}
+                headers['Range'] = 'bytes=%s' % (fragment_range,)
             for count in itertools.count():
                 try:
-                    success, frag_content = self._download_fragment(ctx, fragment_url, info_dict)
+                    success, frag_content = self._download_fragment(ctx, fragment_url, info_dict, headers)
                     if not success:
                         return False
                     self._append_fragment(ctx, frag_content)

From 4eaeb9b2c680ed097770ce976c3b37a1b05c0800 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 27 Jan 2024 15:37:08 +0000
Subject: [PATCH 724/860] [InfoExtractor] Support byte range for DASH * adapted
 from https://github.com/ytdl-org/youtube-dl/pull/30279 * thx former GH user
 kikuyan

---
 test/test_InfoExtractor.py          |  66 ++++++
 test/testdata/mpd/range_only.mpd    |  35 +++
 test/testdata/mpd/subtitles.mpd     | 351 ++++++++++++++++++++++++++++
 test/testdata/mpd/url_and_range.mpd |  32 +++
 youtube_dl/extractor/common.py      | 114 ++++++---
 5 files changed, 562 insertions(+), 36 deletions(-)
 create mode 100644 test/testdata/mpd/range_only.mpd
 create mode 100644 test/testdata/mpd/subtitles.mpd
 create mode 100644 test/testdata/mpd/url_and_range.mpd

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index 043b62243..d55d6ad54 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -1126,6 +1126,72 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/
                     'fps': 30,
                 }],
                 {},
+            ), (
+                # https://github.com/ytdl-org/youtube-dl/issues/30235
+                # Bento4 generated test mpd
+                # mp4dash --mpd-name=manifest.mpd --no-split --use-segment-list mediafiles
+                'url_and_range',
+                'http://unknown/manifest.mpd',  # mpd_url
+                'http://unknown/',  # mpd_base_url
+                [{
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'fragment_base_url': 'http://unknown/',
+                    'ext': 'm4a',
+                    'format_id': 'audio-und-mp4a.40.2',
+                    'format_note': 'DASH audio',
+                    'container': 'm4a_dash',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'none',
+                    'tbr': 98.808,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'fragment_base_url': 'http://unknown/',
+                    'ext': 'mp4',
+                    'format_id': 'video-avc1',
+                    'format_note': 'DASH video',
+                    'container': 'mp4_dash',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'none',
+                    'vcodec': 'avc1.4D401E',
+                    'tbr': 699.597,
+                    'width': 768,
+                    'height': 432
+                }],
+                {},
+            ), (
+                # https://github.com/ytdl-org/youtube-dl/issues/27575
+                # GPAC generated test mpd
+                # MP4Box -dash 10000 -single-file -out manifest.mpd mediafiles
+                'range_only',
+                'http://unknown/manifest.mpd',  # mpd_url
+                'http://unknown/',  # mpd_base_url
+                [{
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'fragment_base_url': 'http://unknown/audio_dashinit.mp4',
+                    'ext': 'm4a',
+                    'format_id': '2',
+                    'format_note': 'DASH audio',
+                    'container': 'm4a_dash',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'mp4a.40.2',
+                    'vcodec': 'none',
+                    'tbr': 98.096,
+                }, {
+                    'manifest_url': 'http://unknown/manifest.mpd',
+                    'fragment_base_url': 'http://unknown/video_dashinit.mp4',
+                    'ext': 'mp4',
+                    'format_id': '1',
+                    'format_note': 'DASH video',
+                    'container': 'mp4_dash',
+                    'protocol': 'http_dash_segments',
+                    'acodec': 'none',
+                    'vcodec': 'avc1.4D401E',
+                    'tbr': 526.987,
+                    'width': 768,
+                    'height': 432
+                }],
+                {},
             ), (
                 'subtitles',
                 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd',
diff --git a/test/testdata/mpd/range_only.mpd b/test/testdata/mpd/range_only.mpd
new file mode 100644
index 000000000..e0c2152d1
--- /dev/null
+++ b/test/testdata/mpd/range_only.mpd
@@ -0,0 +1,35 @@
+<?xml version="1.0"?>
+<!-- MPD file Generated with GPAC version 1.0.1-revrelease at 2021-11-27T20:53:11.690Z -->
+<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" minBufferTime="PT1.500S" type="static" mediaPresentationDuration="PT0H0M30.196S" maxSegmentDuration="PT0H0M10.027S" profiles="urn:mpeg:dash:profile:full:2011">
+ <ProgramInformation moreInformationURL="http://gpac.io">
+  <Title>manifest.mpd generated by GPAC</Title>
+ </ProgramInformation>
+
+ <Period duration="PT0H0M30.196S">
+  <AdaptationSet segmentAlignment="true" maxWidth="768" maxHeight="432" maxFrameRate="30000/1001" par="16:9" lang="und" startWithSAP="1">
+   <Representation id="1" mimeType="video/mp4" codecs="avc1.4D401E" width="768" height="432" frameRate="30000/1001" sar="1:1" bandwidth="526987">
+    <BaseURL>video_dashinit.mp4</BaseURL>
+    <SegmentList timescale="90000" duration="900000">
+     <Initialization range="0-881"/>
+     <SegmentURL mediaRange="882-876094" indexRange="882-925"/>
+     <SegmentURL mediaRange="876095-1466732" indexRange="876095-876138"/>
+     <SegmentURL mediaRange="1466733-1953615" indexRange="1466733-1466776"/>
+     <SegmentURL mediaRange="1953616-1994211" indexRange="1953616-1953659"/>
+    </SegmentList>
+   </Representation>
+  </AdaptationSet>
+  <AdaptationSet segmentAlignment="true" lang="und" startWithSAP="1">
+   <Representation id="2" mimeType="audio/mp4" codecs="mp4a.40.2" audioSamplingRate="48000" bandwidth="98096">
+    <AudioChannelConfiguration schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011" value="2"/>
+    <BaseURL>audio_dashinit.mp4</BaseURL>
+    <SegmentList timescale="48000" duration="480000">
+     <Initialization range="0-752"/>
+     <SegmentURL mediaRange="753-124129" indexRange="753-796"/>
+     <SegmentURL mediaRange="124130-250544" indexRange="124130-124173"/>
+     <SegmentURL mediaRange="250545-374929" indexRange="250545-250588"/>
+    </SegmentList>
+   </Representation>
+  </AdaptationSet>
+ </Period>
+</MPD>
+
diff --git a/test/testdata/mpd/subtitles.mpd b/test/testdata/mpd/subtitles.mpd
new file mode 100644
index 000000000..6f948adba
--- /dev/null
+++ b/test/testdata/mpd/subtitles.mpd
@@ -0,0 +1,351 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!-- Created with Unified Streaming Platform (version=1.10.18-20255) -->
+<MPD
+  xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+  xmlns="urn:mpeg:dash:schema:mpd:2011"
+  xsi:schemaLocation="urn:mpeg:dash:schema:mpd:2011 http://standards.iso.org/ittf/PubliclyAvailableStandards/MPEG-DASH_schema_files/DASH-MPD.xsd"
+  type="static"
+  mediaPresentationDuration="PT14M48S"
+  maxSegmentDuration="PT1M"
+  minBufferTime="PT10S"
+  profiles="urn:mpeg:dash:profile:isoff-live:2011">
+  <Period
+    id="1"
+    duration="PT14M48S">
+    <BaseURL>dash/</BaseURL>
+    <AdaptationSet
+      id="1"
+      group="1"
+      contentType="audio"
+      segmentAlignment="true"
+      audioSamplingRate="48000"
+      mimeType="audio/mp4"
+      codecs="mp4a.40.2"
+      startWithSAP="1">
+      <AudioChannelConfiguration
+        schemeIdUri="urn:mpeg:dash:23003:3:audio_channel_configuration:2011"
+        value="2" />
+      <Role schemeIdUri="urn:mpeg:dash:role:2011" value="main" />
+      <SegmentTemplate
+        timescale="48000"
+        initialization="3144-kZT4LWMQw6Rh7Kpd-$RepresentationID$.dash"
+        media="3144-kZT4LWMQw6Rh7Kpd-$RepresentationID$-$Time$.dash">
+        <SegmentTimeline>
+          <S t="0" d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="96256" r="2" />
+          <S d="95232" />
+          <S d="3584" />
+        </SegmentTimeline>
+      </SegmentTemplate>
+      <Representation
+        id="audio=128001"
+        bandwidth="128001">
+      </Representation>
+    </AdaptationSet>
+    <AdaptationSet
+      id="2"
+      group="3"
+      contentType="text"
+      lang="en"
+      mimeType="application/mp4"
+      codecs="stpp"
+      startWithSAP="1">
+      <Role schemeIdUri="urn:mpeg:dash:role:2011" value="subtitle" />
+      <SegmentTemplate
+        timescale="1000"
+        initialization="3144-kZT4LWMQw6Rh7Kpd-$RepresentationID$.dash"
+        media="3144-kZT4LWMQw6Rh7Kpd-$RepresentationID$-$Time$.dash">
+        <SegmentTimeline>
+          <S t="0" d="60000" r="9" />
+          <S d="24000" />
+        </SegmentTimeline>
+      </SegmentTemplate>
+      <Representation
+        id="textstream_eng=1000"
+        bandwidth="1000">
+      </Representation>
+    </AdaptationSet>
+    <AdaptationSet
+      id="3"
+      group="2"
+      contentType="video"
+      par="960:409"
+      minBandwidth="100000"
+      maxBandwidth="4482000"
+      maxWidth="1689"
+      maxHeight="720"
+      segmentAlignment="true"
+      mimeType="video/mp4"
+      codecs="avc1.4D401F"
+      startWithSAP="1">
+      <Role schemeIdUri="urn:mpeg:dash:role:2011" value="main" />
+      <SegmentTemplate
+        timescale="12288"
+        initialization="3144-kZT4LWMQw6Rh7Kpd-$RepresentationID$.dash"
+        media="3144-kZT4LWMQw6Rh7Kpd-$RepresentationID$-$Time$.dash">
+        <SegmentTimeline>
+          <S t="0" d="24576" r="443" />
+        </SegmentTimeline>
+      </SegmentTemplate>
+      <Representation
+        id="video=100000"
+        bandwidth="100000"
+        width="336"
+        height="144"
+        sar="2880:2863"
+        scanType="progressive">
+      </Representation>
+      <Representation
+        id="video=326000"
+        bandwidth="326000"
+        width="562"
+        height="240"
+        sar="115200:114929"
+        scanType="progressive">
+      </Representation>
+      <Representation
+        id="video=698000"
+        bandwidth="698000"
+        width="844"
+        height="360"
+        sar="86400:86299"
+        scanType="progressive">
+      </Representation>
+      <Representation
+        id="video=1493000"
+        bandwidth="1493000"
+        width="1126"
+        height="480"
+        sar="230400:230267"
+        scanType="progressive">
+      </Representation>
+      <Representation
+        id="video=4482000"
+        bandwidth="4482000"
+        width="1688"
+        height="720"
+        sar="86400:86299"
+        scanType="progressive">
+      </Representation>
+    </AdaptationSet>
+  </Period>
+</MPD>
diff --git a/test/testdata/mpd/url_and_range.mpd b/test/testdata/mpd/url_and_range.mpd
new file mode 100644
index 000000000..b8c68aad2
--- /dev/null
+++ b/test/testdata/mpd/url_and_range.mpd
@@ -0,0 +1,32 @@
+<?xml version="1.0" ?>
+<MPD xmlns="urn:mpeg:dash:schema:mpd:2011" profiles="urn:mpeg:dash:profile:isoff-live:2011" minBufferTime="PT10.01S" mediaPresentationDuration="PT30.097S" type="static">
+  <!-- Created with Bento4 mp4-dash.py, VERSION=2.0.0-639 -->
+  <Period>
+    <!-- Video -->
+    <AdaptationSet mimeType="video/mp4" segmentAlignment="true" startWithSAP="1" maxWidth="768" maxHeight="432">
+      <Representation id="video-avc1" codecs="avc1.4D401E" width="768" height="432" scanType="progressive" frameRate="30000/1001" bandwidth="699597">
+        <SegmentList timescale="1000" duration="10010">
+          <Initialization sourceURL="video-frag.mp4" range="36-746"/>
+          <SegmentURL media="video-frag.mp4" mediaRange="747-876117"/>
+          <SegmentURL media="video-frag.mp4" mediaRange="876118-1466913"/>
+          <SegmentURL media="video-frag.mp4" mediaRange="1466914-1953954"/>
+          <SegmentURL media="video-frag.mp4" mediaRange="1953955-1994652"/>
+        </SegmentList>
+      </Representation>
+    </AdaptationSet>
+    <!-- Audio -->
+    <AdaptationSet mimeType="audio/mp4" startWithSAP="1" segmentAlignment="true">
+      <Representation id="audio-und-mp4a.40.2" codecs="mp4a.40.2" bandwidth="98808" audioSamplingRate="48000">
+        <AudioChannelConfiguration schemeIdUri="urn:mpeg:mpegB:cicp:ChannelConfiguration" value="2"/>
+        <SegmentList timescale="1000" duration="10010">
+          <Initialization sourceURL="audio-frag.mp4" range="32-623"/>
+          <SegmentURL media="audio-frag.mp4" mediaRange="624-124199"/>
+          <SegmentURL media="audio-frag.mp4" mediaRange="124200-250303"/>
+          <SegmentURL media="audio-frag.mp4" mediaRange="250304-374365"/>
+          <SegmentURL media="audio-frag.mp4" mediaRange="374366-374836"/>
+        </SegmentList>
+      </Representation>
+    </AdaptationSet>
+  </Period>
+</MPD>
+
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index ed55d3e07..76414554a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -183,6 +183,8 @@ class InfoExtractor(object):
                                             fragment_base_url
                                  * "duration" (optional, int or float)
                                  * "filesize" (optional, int)
+                                 * "range" (optional, str of the form "start-end"
+                                            to use in HTTP Range header)
                     * preference Order number of this format. If this field is
                                  present and not None, the formats get sorted
                                  by this field, regardless of all other values.
@@ -2296,15 +2298,27 @@ class InfoExtractor(object):
             def extract_Initialization(source):
                 initialization = source.find(_add_ns('Initialization'))
                 if initialization is not None:
-                    ms_info['initialization_url'] = initialization.attrib['sourceURL']
+                    ms_info['initialization_url'] = initialization.get('sourceURL') or base_url
+                    initialization_url_range = initialization.get('range')
+                    if initialization_url_range:
+                        ms_info['initialization_url_range'] = initialization_url_range
 
             segment_list = element.find(_add_ns('SegmentList'))
             if segment_list is not None:
                 extract_common(segment_list)
                 extract_Initialization(segment_list)
                 segment_urls_e = segment_list.findall(_add_ns('SegmentURL'))
-                if segment_urls_e:
-                    ms_info['segment_urls'] = [segment.attrib['media'] for segment in segment_urls_e]
+                segment_urls = traverse_obj(segment_urls_e, (
+                    Ellipsis, T(lambda e: e.attrib), 'media'))
+                if segment_urls:
+                    ms_info['segment_urls'] = segment_urls
+                segment_urls_range = traverse_obj(segment_urls_e, (
+                    Ellipsis, T(lambda e: e.attrib), 'mediaRange',
+                    T(lambda r: re.findall(r'^\d+-\d+$', r)), 0))
+                if segment_urls_range:
+                    ms_info['segment_urls_range'] = segment_urls_range
+                    if not segment_urls:
+                        ms_info['segment_urls'] = [base_url for _ in segment_urls_range]
             else:
                 segment_template = element.find(_add_ns('SegmentTemplate'))
                 if segment_template is not None:
@@ -2443,6 +2457,11 @@ class InfoExtractor(object):
                     def location_key(location):
                         return 'url' if re.match(r'^https?://', location) else 'path'
 
+                    def calc_segment_duration():
+                        return float_or_none(
+                            representation_ms_info['segment_duration'],
+                            representation_ms_info['timescale']) if 'segment_duration' in representation_ms_info else None
+
                     if 'segment_urls' not in representation_ms_info and 'media' in representation_ms_info:
 
                         media_template = prepare_template('media', ('Number', 'Bandwidth', 'Time'))
@@ -2512,45 +2531,68 @@ class InfoExtractor(object):
                                         'duration': duration,
                                     })
                                     segment_index += 1
-                            representation_ms_info['fragments'] = fragments
-                        elif 'segment_urls' in representation_ms_info:
+                        elif 'segment_urls_range' in representation_ms_info:
+                            # Segment URLs with mediaRange
+                            # Example: https://kinescope.io/200615537/master.mpd
+                            # https://github.com/ytdl-org/youtube-dl/issues/30235
+                            # or any mpd generated with Bento4 `mp4dash --no-split --use-segment-list`
+                            segment_duration = calc_segment_duration()
+                            for segment_url, segment_url_range in zip(
+                                    representation_ms_info['segment_urls'], representation_ms_info['segment_urls_range']):
+                                fragments.append({
+                                    location_key(segment_url): segment_url,
+                                    'range': segment_url_range,
+                                    'duration': segment_duration,
+                                })
+                        else:
                             # Segment URLs with no SegmentTimeline
                             # Example: https://www.seznam.cz/zpravy/clanek/cesko-zasahne-vitr-o-sile-vichrice-muze-byt-i-zivotu-nebezpecny-39091
                             # https://github.com/ytdl-org/youtube-dl/pull/14844
-                            fragments = []
-                            segment_duration = float_or_none(
-                                representation_ms_info['segment_duration'],
-                                representation_ms_info['timescale']) if 'segment_duration' in representation_ms_info else None
+                            segment_duration = calc_segment_duration()
                             for segment_url in representation_ms_info['segment_urls']:
-                                fragment = {
+                                fragments.append({
                                     location_key(segment_url): segment_url,
-                                }
-                                if segment_duration:
-                                    fragment['duration'] = segment_duration
-                                fragments.append(fragment)
-                            representation_ms_info['fragments'] = fragments
-                        # If there is a fragments key available then we correctly recognized fragmented media.
-                        # Otherwise we will assume unfragmented media with direct access. Technically, such
-                        # assumption is not necessarily correct since we may simply have no support for
-                        # some forms of fragmented media renditions yet, but for now we'll use this fallback.
-                        if 'fragments' in representation_ms_info:
-                            f.update({
-                                # NB: mpd_url may be empty when MPD manifest is parsed from a string
-                                'url': mpd_url or base_url,
-                                'fragment_base_url': base_url,
-                                'fragments': [],
-                                'protocol': 'http_dash_segments',
+                                    'duration': segment_duration,
+                                })
+                        representation_ms_info['fragments'] = fragments
+
+                    # If there is a fragments key available then we correctly recognized fragmented media.
+                    # Otherwise we will assume unfragmented media with direct access. Technically, such
+                    # assumption is not necessarily correct since we may simply have no support for
+                    # some forms of fragmented media renditions yet, but for now we'll use this fallback.
+                    if 'fragments' in representation_ms_info:
+                        base_url = representation_ms_info['base_url'] 
+                        f.update({
+                            # NB: mpd_url may be empty when MPD manifest is parsed from a string
+                            'url': mpd_url or base_url,
+                            'fragment_base_url': base_url,
+                            'fragments': [],
+                            'protocol': 'http_dash_segments',
+                        })
+                        if 'initialization_url' in representation_ms_info and 'initialization_url_range' in representation_ms_info:
+                            # Initialization URL with range (accompanied by Segment URLs with mediaRange above)
+                            # https://github.com/ytdl-org/youtube-dl/issues/30235
+                            initialization_url = representation_ms_info['initialization_url']
+                            f['fragments'].append({
+                                location_key(initialization_url): initialization_url,
+                                'range': representation_ms_info['initialization_url_range'],
                             })
-                            if 'initialization_url' in representation_ms_info:
-                                initialization_url = representation_ms_info['initialization_url']
-                                if not f.get('url'):
-                                    f['url'] = initialization_url
-                                f['fragments'].append({location_key(initialization_url): initialization_url})
-                            f['fragments'].extend(representation_ms_info['fragments'])
-                        else:
-                            # Assuming direct URL to unfragmented media.
-                            f['url'] = base_url
-                        formats.append(f)
+                        elif 'initialization_url' in representation_ms_info:
+                            initialization_url = representation_ms_info['initialization_url']
+                            if not f.get('url'):
+                                f['url'] = initialization_url
+                            f['fragments'].append({location_key(initialization_url): initialization_url})
+                        elif 'initialization_url_range' in representation_ms_info:
+                            # no Initialization URL but range (accompanied by no Segment URLs but mediaRange above)
+                            # https://github.com/ytdl-org/youtube-dl/issues/27575
+                            f['fragments'].append({
+                                location_key(base_url): base_url,
+                                'range': representation_ms_info['initialization_url_range'],
+                            })
+                        f['fragments'].extend(representation_ms_info['fragments'])
+                        if not period_duration:
+                            period_duration = sum(traverse_obj(representation_ms_info, (
+                                'fragments', Ellipsis, 'duration', T(float_or_none))))
                     else:
                         # Assuming direct URL to unfragmented media.
                         f['url'] = representation_ms_info['base_url']

From 1fd8f802b836cc1e8bf87b2dbe02decd6a980cb6 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 27 Jan 2024 15:45:43 +0000
Subject: [PATCH 725/860] [InfoExtractor] Correctly resolve BaseURL in DASH
 manifest Specs: * ISO/IEC 23009-1:2012 section 5.6 * RFC 3986 section 5.

---
 youtube_dl/extractor/common.py | 21 +++++++++++++++++++--
 1 file changed, 19 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 76414554a..0e5dfd8fa 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -2262,9 +2262,24 @@ class InfoExtractor(object):
         def is_drm_protected(element):
             return element.find(_add_ns('ContentProtection')) is not None
 
+        from ..utils import YoutubeDLHandler
+        fix_path = YoutubeDLHandler._fix_path
+
+        def resolve_base_url(element, parent_base_url=None):
+            # TODO: use native XML traversal when ready
+            b_url = traverse_obj(element, (
+                T(lambda e: e.find(_add_ns('BaseURL')).text)))
+            if parent_base_url and b_url:
+                if not parent_base_url[-1] in ('/', ':'):
+                    parent_base_url += '/'
+                b_url = compat_urlparse.urljoin(parent_base_url, b_url)
+            if b_url:
+                b_url = fix_path(b_url)
+            return b_url or parent_base_url
+
         def extract_multisegment_info(element, ms_parent_info):
             ms_info = ms_parent_info.copy()
-            base_url = ms_info.get('base_url')
+            base_url = ms_info['base_url'] = resolve_base_url(element, ms_info.get('base_url'))
 
             # As per [1, 5.3.9.2.2] SegmentList and SegmentTemplate share some
             # common attributes and elements.  We will only extract relevant
@@ -2336,11 +2351,13 @@ class InfoExtractor(object):
         mpd_duration = parse_duration(mpd_doc.get('mediaPresentationDuration'))
         formats, subtitles = [], {}
         stream_numbers = collections.defaultdict(int)
+        mpd_base_url = resolve_base_url(mpd_doc, mpd_base_url or mpd_url)
         for period in mpd_doc.findall(_add_ns('Period')):
             period_duration = parse_duration(period.get('duration')) or mpd_duration
             period_ms_info = extract_multisegment_info(period, {
                 'start_number': 1,
                 'timescale': 1,
+                'base_url': mpd_base_url,
             })
             for adaptation_set in period.findall(_add_ns('AdaptationSet')):
                 if is_drm_protected(adaptation_set):
@@ -2561,7 +2578,7 @@ class InfoExtractor(object):
                     # assumption is not necessarily correct since we may simply have no support for
                     # some forms of fragmented media renditions yet, but for now we'll use this fallback.
                     if 'fragments' in representation_ms_info:
-                        base_url = representation_ms_info['base_url'] 
+                        base_url = representation_ms_info['base_url']
                         f.update({
                             # NB: mpd_url may be empty when MPD manifest is parsed from a string
                             'url': mpd_url or base_url,

From bdda6b81df61f52eed2502c8ae624d297d918488 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 27 Jan 2024 15:57:25 +0000
Subject: [PATCH 726/860] [Vbox7IE] Improve extraction * DASH extraction no
 longer fails with new range support * but always find combined formats if
 available * suppress ineffective XFF geo-bypass (causes time-outs) * adapted
 from https://github.com/ytdl-org/youtube-dl/pull/29680 * thx former GH user
 kikuyan

---
 youtube_dl/extractor/vbox7.py | 121 ++++++++++++++++++++++++----------
 1 file changed, 86 insertions(+), 35 deletions(-)

diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index 8152acefd..c504c5311 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -2,9 +2,19 @@
 from __future__ import unicode_literals
 
 import re
+import time
 
 from .common import InfoExtractor
-from ..utils import ExtractorError
+from ..utils import (
+    determine_ext,
+    ExtractorError,
+    float_or_none,
+    merge_dicts,
+    T,
+    traverse_obj,
+    txt_or_none,
+    url_or_none,
+)
 
 
 class Vbox7IE(InfoExtractor):
@@ -20,10 +30,12 @@ class Vbox7IE(InfoExtractor):
                         )
                         (?P<id>[\da-fA-F]+)
                     '''
+    _EMBED_REGEX = [r'<iframe[^>]+src=(?P<q>["\'])(?P<url>(?:https?:)?//vbox7\.com/emb/external\.php.+?)(?P=q)']
     _GEO_COUNTRIES = ['BG']
+    _GEO_BYPASS = False
     _TESTS = [{
-        'url': 'http://vbox7.com/play:0946fff23c',
-        'md5': 'a60f9ab3a3a2f013ef9a967d5f7be5bf',
+        'url': 'https://vbox7.com/play:0946fff23c',
+        'md5': '50ca1f78345a9c15391af47d8062d074',
         'info_dict': {
             'id': '0946fff23c',
             'ext': 'mp4',
@@ -34,18 +46,21 @@ class Vbox7IE(InfoExtractor):
             'upload_date': '20160812',
             'uploader': 'zdraveibulgaria',
         },
-        'params': {
-            'proxy': '127.0.0.1:8118',
-        },
+        'expected_warnings': [
+            'Unable to download webpage',
+        ],
     }, {
         'url': 'http://vbox7.com/play:249bb972c2',
-        'md5': '99f65c0c9ef9b682b97313e052734c3f',
+        'md5': 'aaf19465e37ec0b30b918df83ec32c50',
         'info_dict': {
             'id': '249bb972c2',
             'ext': 'mp4',
             'title': 'Смях! Чудо - чист за секунди - Скрита камера',
+            'description': 'Смях! Чудо - чист за секунди - Скрита камера',
+            'timestamp': 1360215023,
+            'upload_date': '20130207',
+            'uploader': 'svideteliat_ot_varshava',
         },
-        'skip': 'georestricted',
     }, {
         'url': 'http://vbox7.com/emb/external.php?vid=a240d20f9c&autoplay=1',
         'only_matching': True,
@@ -54,52 +69,88 @@ class Vbox7IE(InfoExtractor):
         'only_matching': True,
     }]
 
-    @staticmethod
-    def _extract_url(webpage):
-        mobj = re.search(
-            r'<iframe[^>]+src=(?P<q>["\'])(?P<url>(?:https?:)?//vbox7\.com/emb/external\.php.+?)(?P=q)',
-            webpage)
+    @classmethod
+    def _extract_url(cls, webpage):
+        mobj = re.search(cls._EMBED_REGEX[0], webpage)
         if mobj:
             return mobj.group('url')
 
     def _real_extract(self, url):
         video_id = self._match_id(url)
+        url = 'https://vbox7.com/play:%s' % (video_id,)
 
+        now = time.time()
         response = self._download_json(
-            'https://www.vbox7.com/ajax/video/nextvideo.php?vid=%s' % video_id,
-            video_id)
+            'https://www.vbox7.com/aj/player/item/options?vid=%s' % (video_id,),
+            video_id, headers={'Referer': url})
+        # estimate time to which possible `ago` member is relative
+        now = now + 0.5 * (time.time() - now)
 
         if 'error' in response:
             raise ExtractorError(
                 '%s said: %s' % (self.IE_NAME, response['error']), expected=True)
 
-        video = response['options']
+        video_url = traverse_obj(response, ('options', 'src', T(url_or_none)))
 
-        title = video['title']
-        video_url = video['src']
-
-        if '/na.mp4' in video_url:
+        if '/na.mp4' in video_url or '':
             self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
 
-        uploader = video.get('uploader')
+        ext = determine_ext(video_url)
+        if ext == 'mpd':
+            # In case MPD cannot be parsed, or anyway, get mp4 combined
+            # formats usually provided to Safari, iOS, and old Windows
+            try:
+                formats, subtitles = self._extract_mpd_formats_and_subtitles(
+                    video_url, video_id, 'dash', fatal=False)
+            except KeyError:
+                self.report_warning('Failed to parse MPD manifest')
+                formats, subtitles = [], {}
 
-        webpage = self._download_webpage(
-            'http://vbox7.com/play:%s' % video_id, video_id, fatal=None)
+            video = response['options']
+            resolutions = (1080, 720, 480, 240, 144)
+            highest_res = traverse_obj(video, ('highestRes', T(int))) or resolutions[0]
+            for res in traverse_obj(video, ('resolutions', lambda _, r: int(r) > 0)) or resolutions:
+                if res > highest_res:
+                    continue
+                formats.append({
+                    'url': video_url.replace('.mpd', '_%d.mp4' % res),
+                    'format_id': '%dp' % res,
+                    'height': res,
+                })
+            # if above formats are flaky, enable the line below
+            # self._check_formats(formats, video_id)
+        else:
+            formats = [{
+                'url': video_url,
+            }]
+            subtitles = {}
+        self._sort_formats(formats)
 
-        info = {}
+        webpage = self._download_webpage(url, video_id, fatal=False) or ''
 
-        if webpage:
-            info = self._search_json_ld(
-                webpage.replace('"/*@context"', '"@context"'), video_id,
-                fatal=False)
+        info = self._search_json_ld(
+            webpage.replace('"/*@context"', '"@context"'), video_id,
+            fatal=False) if webpage else {}
 
-        info.update({
+        if not info.get('title'):
+            info['title'] = traverse_obj(response, (
+                'options', 'title', T(txt_or_none))) or self._og_search_title(webpage)
+
+        def if_missing(k):
+            return lambda x: None if k in info else x
+
+        info = merge_dicts(info, {
             'id': video_id,
-            'title': title,
-            'url': video_url,
-            'uploader': uploader,
-            'thumbnail': self._proto_relative_url(
+            'formats': formats,
+            'subtitles': subtitles or None,
+        }, info, traverse_obj(response, ('options', {
+            'uploader': ('uploader', T(txt_or_none)),
+            'timestamp': ('ago', T(if_missing('timestamp')), T(lambda t: int(round((now - t) / 60.0)) * 60)),
+            'duration': ('duration', T(if_missing('duration')), T(float_or_none)),
+        })))
+        if 'thumbnail' not in info:
+            info['thumbnail'] = self._proto_relative_url(
                 info.get('thumbnail') or self._og_search_thumbnail(webpage),
-                'http:'),
-        })
+                'https:'),
+
         return info

From 4416f82c809a81737d68875dcb201e366d58dabd Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 27 Jan 2024 18:17:09 +0000
Subject: [PATCH 727/860] [Vbox7IE] Sanitise ld+json containing unexpected
 characters * based on PR #29680 * added hack to force invoking
 `transform_source` * fixes #26218

---
 youtube_dl/extractor/vbox7.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index c504c5311..d114ecb07 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -5,6 +5,7 @@ import re
 import time
 
 from .common import InfoExtractor
+from ..compat import compat_kwargs
 from ..utils import (
     determine_ext,
     ExtractorError,
@@ -75,6 +76,27 @@ class Vbox7IE(InfoExtractor):
         if mobj:
             return mobj.group('url')
 
+    # transform_source=None, fatal=True
+    def _parse_json(self, json_string, video_id, *args, **kwargs):
+        if '"@context"' in json_string[:30]:
+            # this is ld+json, or that's the way to bet
+            transform_source = args[0] if len(args) > 0 else kwargs.get('transform_source')
+            if not transform_source:
+
+                def fix_chars(src):
+                    # fix malformed ld+json: replace raw CRLFs with escaped LFs
+                    return re.sub(
+                        r'"[^"]+"', lambda m: re.sub(r'\r?\n', r'\\n', m.group(0)), src)
+
+                if len(args) > 0:
+                    args = (fix_chars,) + args[1:]
+                else:
+                    kwargs['transform_source'] = fix_chars
+                    kwargs = compat_kwargs(kwargs)
+
+        return super(Vbox7IE, self)._parse_json(
+            json_string, video_id, *args, **kwargs)
+
     def _real_extract(self, url):
         video_id = self._match_id(url)
         url = 'https://vbox7.com/play:%s' % (video_id,)

From 7687389f08a5c7c49e57d1b7f7b11b1c87b47b68 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 18 Feb 2024 20:55:01 +0000
Subject: [PATCH 728/860] [Vbox7] Improve extraction, adding features from
 yt-dlp PR #9100

* changes from https://github.com/yt-dlp/yt-dlp/pull/9100 (thx
seproDev):
  - attempt HLS extraction
  - re-enable XFF
  - test `view_count`, `duration` extraction
* improve commenting, error checks
---
 youtube_dl/extractor/vbox7.py | 80 +++++++++++++++++++++++------------
 1 file changed, 53 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/extractor/vbox7.py b/youtube_dl/extractor/vbox7.py
index d114ecb07..1c0b77074 100644
--- a/youtube_dl/extractor/vbox7.py
+++ b/youtube_dl/extractor/vbox7.py
@@ -7,6 +7,7 @@ import time
 from .common import InfoExtractor
 from ..compat import compat_kwargs
 from ..utils import (
+    base_url,
     determine_ext,
     ExtractorError,
     float_or_none,
@@ -14,6 +15,7 @@ from ..utils import (
     T,
     traverse_obj,
     txt_or_none,
+    url_basename,
     url_or_none,
 )
 
@@ -33,8 +35,8 @@ class Vbox7IE(InfoExtractor):
                     '''
     _EMBED_REGEX = [r'<iframe[^>]+src=(?P<q>["\'])(?P<url>(?:https?:)?//vbox7\.com/emb/external\.php.+?)(?P=q)']
     _GEO_COUNTRIES = ['BG']
-    _GEO_BYPASS = False
     _TESTS = [{
+        # the http: URL just redirects here
         'url': 'https://vbox7.com/play:0946fff23c',
         'md5': '50ca1f78345a9c15391af47d8062d074',
         'info_dict': {
@@ -42,17 +44,19 @@ class Vbox7IE(InfoExtractor):
             'ext': 'mp4',
             'title': 'Борисов: Притеснен съм за бъдещето на България',
             'description': 'По думите му е опасно страната ни да бъде обявена за "сигурна"',
-            'thumbnail': r're:^https?://.*\.jpg$',
             'timestamp': 1470982814,
             'upload_date': '20160812',
             'uploader': 'zdraveibulgaria',
+            'thumbnail': r're:^https?://.*\.jpg$',
+            'view_count': int,
+            'duration': 2640,
         },
         'expected_warnings': [
             'Unable to download webpage',
         ],
     }, {
         'url': 'http://vbox7.com/play:249bb972c2',
-        'md5': 'aaf19465e37ec0b30b918df83ec32c50',
+        'md5': '99f65c0c9ef9b682b97313e052734c3f',
         'info_dict': {
             'id': '249bb972c2',
             'ext': 'mp4',
@@ -61,7 +65,11 @@ class Vbox7IE(InfoExtractor):
             'timestamp': 1360215023,
             'upload_date': '20130207',
             'uploader': 'svideteliat_ot_varshava',
+            'thumbnail': 'https://i49.vbox7.com/o/249/249bb972c20.jpg',
+            'view_count': int,
+            'duration': 83,
         },
+        'expected_warnings': ['Failed to download m3u8 information'],
     }, {
         'url': 'http://vbox7.com/emb/external.php?vid=a240d20f9c&autoplay=1',
         'only_matching': True,
@@ -76,6 +84,9 @@ class Vbox7IE(InfoExtractor):
         if mobj:
             return mobj.group('url')
 
+    # specialisation to transform what looks like ld+json that
+    # may contain invalid character combinations
+
     # transform_source=None, fatal=True
     def _parse_json(self, json_string, video_id, *args, **kwargs):
         if '"@context"' in json_string[:30]:
@@ -103,49 +114,64 @@ class Vbox7IE(InfoExtractor):
 
         now = time.time()
         response = self._download_json(
-            'https://www.vbox7.com/aj/player/item/options?vid=%s' % (video_id,),
-            video_id, headers={'Referer': url})
+            'https://www.vbox7.com/aj/player/item/options', video_id,
+            query={'vid': video_id}, headers={'Referer': url})
         # estimate time to which possible `ago` member is relative
         now = now + 0.5 * (time.time() - now)
 
-        if 'error' in response:
+        if traverse_obj(response, 'error'):
             raise ExtractorError(
                 '%s said: %s' % (self.IE_NAME, response['error']), expected=True)
 
-        video_url = traverse_obj(response, ('options', 'src', T(url_or_none)))
+        src_url = traverse_obj(response, ('options', 'src', T(url_or_none))) or ''
 
-        if '/na.mp4' in video_url or '':
+        fmt_base = url_basename(src_url).rsplit('.', 1)[0].rsplit('_', 1)[0]
+        if fmt_base in ('na', 'vn'):
             self.raise_geo_restricted(countries=self._GEO_COUNTRIES)
 
-        ext = determine_ext(video_url)
+        ext = determine_ext(src_url)
         if ext == 'mpd':
-            # In case MPD cannot be parsed, or anyway, get mp4 combined
-            # formats usually provided to Safari, iOS, and old Windows
+            # extract MPD
             try:
                 formats, subtitles = self._extract_mpd_formats_and_subtitles(
-                    video_url, video_id, 'dash', fatal=False)
-            except KeyError:
+                    src_url, video_id, 'dash', fatal=False)
+            except KeyError:  # fatal doesn't catch this
                 self.report_warning('Failed to parse MPD manifest')
                 formats, subtitles = [], {}
+        elif ext != 'm3u8':
+            formats = [{
+                'url': src_url,
+            }] if src_url else []
+            subtitles = {}
 
+        if src_url:
+            # possibly extract HLS, based on https://github.com/yt-dlp/yt-dlp/pull/9100
+            fmt_base = base_url(src_url) + fmt_base
+            # prepare for _extract_m3u8_formats_and_subtitles()
+            # hls_formats, hls_subs = self._extract_m3u8_formats_and_subtitles(
+            hls_formats = self._extract_m3u8_formats(
+                '{0}.m3u8'.format(fmt_base), video_id, m3u8_id='hls', fatal=False)
+            formats.extend(hls_formats)
+            # self._merge_subtitles(hls_subs, target=subtitles)
+
+            # In case MPD/HLS cannot be parsed, or anyway, get mp4 combined
+            # formats usually provided to Safari, iOS, and old Windows
             video = response['options']
             resolutions = (1080, 720, 480, 240, 144)
-            highest_res = traverse_obj(video, ('highestRes', T(int))) or resolutions[0]
-            for res in traverse_obj(video, ('resolutions', lambda _, r: int(r) > 0)) or resolutions:
-                if res > highest_res:
-                    continue
-                formats.append({
-                    'url': video_url.replace('.mpd', '_%d.mp4' % res),
-                    'format_id': '%dp' % res,
+            highest_res = traverse_obj(video, (
+                'highestRes', T(int))) or resolutions[0]
+            resolutions = traverse_obj(video, (
+                'resolutions', lambda _, r: highest_res >= int(r) > 0)) or resolutions
+            mp4_formats = traverse_obj(resolutions, (
+                Ellipsis, T(lambda res: {
+                    'url': '{0}_{1}.mp4'.format(fmt_base, res),
+                    'format_id': 'http-{0}'.format(res),
                     'height': res,
-                })
+                })))
             # if above formats are flaky, enable the line below
-            # self._check_formats(formats, video_id)
-        else:
-            formats = [{
-                'url': video_url,
-            }]
-            subtitles = {}
+            # self._check_formats(mp4_formats, video_id)
+            formats.extend(mp4_formats)
+
         self._sort_formats(formats)
 
         webpage = self._download_webpage(url, video_id, fatal=False) or ''

From 48ddab1f3a616d40480818ec9b73eaa27e3a55fa Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 21 Feb 2024 16:29:08 +0000
Subject: [PATCH 729/860] [downloader/external] Fix WgetFD proxy (rev 2)

From PR (defunct source), closes #29343.
Matches https://github.com/yt-dlp/yt-dlp/pull/3152
Thx former user kikuyan.
---
 youtube_dl/downloader/external.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index 7fc864e85..bc228960e 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -206,7 +206,10 @@ class WgetFD(ExternalFD):
                 retry[1] = '0'
             cmd += retry
         cmd += self._option('--bind-address', 'source_address')
-        cmd += self._option('--proxy', 'proxy')
+        proxy = self.params.get('proxy')
+        if proxy:
+            for var in ('http_proxy', 'https_proxy'):
+                cmd += ['--execute', '%s=%s' % (var, proxy)]
         cmd += self._valueless_option('--no-check-certificate', 'nocheckcertificate')
         cmd += self._configuration_args()
         cmd += ['--', info_dict['url']]

From 70f230f9cf28e948662599b6257cb7d1262870e3 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 22 Feb 2024 12:44:00 +0000
Subject: [PATCH 730/860] [GBNews]Add new extractor for GB News TV channel
 (#29432)

* Add extractor for GB News TV channel

* Support more GBNews URL formats
Allow alphanumeric and _ in place of `shows`, which redirect to site's preferred URL

* Update for 2024
---
 youtube_dl/extractor/extractors.py |   1 +
 youtube_dl/extractor/gbnews.py     | 139 +++++++++++++++++++++++++++++
 2 files changed, 140 insertions(+)
 create mode 100644 youtube_dl/extractor/gbnews.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 82221445f..b6d0f42f5 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -443,6 +443,7 @@ from .gamespot import GameSpotIE
 from .gamestar import GameStarIE
 from .gaskrank import GaskrankIE
 from .gazeta import GazetaIE
+from .gbnews import GBNewsIE
 from .gdcvault import GDCVaultIE
 from .gedidigital import GediDigitalIE
 from .generic import GenericIE
diff --git a/youtube_dl/extractor/gbnews.py b/youtube_dl/extractor/gbnews.py
new file mode 100644
index 000000000..f04f30e5a
--- /dev/null
+++ b/youtube_dl/extractor/gbnews.py
@@ -0,0 +1,139 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    extract_attributes,
+    ExtractorError,
+    T,
+    traverse_obj,
+    txt_or_none,
+    url_or_none,
+)
+
+
+class GBNewsIE(InfoExtractor):
+    IE_DESC = 'GB News clips, features and live stream'
+
+    # \w+ is normally shows or news, but apparently any word redirects to the correct URL
+    _VALID_URL = r'https?://(?:www\.)?gbnews\.(?:uk|com)/(?:\w+/)?(?P<id>[^#?]+)'
+
+    _PLATFORM = 'safari'
+    _SSMP_URL = 'https://mm-v2.simplestream.com/ssmp/api.php'
+    _TESTS = [{
+        'url': 'https://www.gbnews.uk/shows/andrew-neils-message-to-companies-choosing-to-boycott-gb-news/106889',
+        'info_dict': {
+            'id': '106889',
+            'ext': 'mp4',
+            'title': "Andrew Neil's message to companies choosing to boycott GB News",
+            'description': 'md5:b281f5d22fd6d5eda64a4e3ba771b351',
+        },
+        'skip': '404 not found',
+    }, {
+        'url': 'https://www.gbnews.com/news/bbc-claudine-gay-harvard-university-antisemitism-row',
+        'info_dict': {
+            'id': '52264136',
+            'display_id': 'bbc-claudine-gay-harvard-university-antisemitism-row',
+            'ext': 'mp4',
+            'title': 'BBC deletes post after furious backlash over headline downplaying antisemitism',
+            'description': 'The post was criticised by former employers of the broadcaster',
+        },
+    }, {
+        'url': 'https://www.gbnews.uk/watchlive',
+        'info_dict': {
+            'id': '1069',
+            'display_id': 'watchlive',
+            'ext': 'mp4',
+            'title': 'GB News Live',
+            'is_live': True,
+        },
+        'params': {
+            'skip_download': 'm3u8',
+        },
+    }]
+
+    def _real_extract(self, url):
+        display_id = self._match_id(url).split('/')[-1]
+
+        webpage = self._download_webpage(url, display_id)
+        # extraction based on https://github.com/ytdl-org/youtube-dl/issues/29341
+        '''
+        <div id="video-106908"
+            class="simplestream"
+            data-id="GB001"
+            data-type="vod"
+            data-key="3Li3Nt2Qs8Ct3Xq9Fi5Uy0Mb2Bj0Qs"
+            data-token="f9c317c727dc07f515b20036c8ef14a6"
+            data-expiry="1624300052"
+            data-uvid="37900558"
+            data-poster="https://thumbnails.simplestreamcdn.com/gbnews/ondemand/37900558.jpg?width=700&"
+            data-npaw="false"
+            data-env="production">
+        '''
+        # exception if no match
+        video_data = self._search_regex(
+            r'(<div\s[^>]*\bclass\s*=\s*(\'|")(?!.*sidebar\b)simplestream(?:\s[\s\w$-]*)?\2[^>]*>)',
+            webpage, 'video data')
+
+        video_data = extract_attributes(video_data)
+        ss_id = video_data.get('data-id')
+        if not ss_id:
+            raise ExtractorError('Simplestream ID not found')
+
+        json_data = self._download_json(
+            self._SSMP_URL, display_id,
+            note='Downloading Simplestream JSON metadata',
+            errnote='Unable to download Simplestream JSON metadata',
+            query={
+                'id': ss_id,
+                'env': video_data.get('data-env', 'production'),
+            }, fatal=False)
+
+        meta_url = traverse_obj(json_data, ('response', 'api_hostname'))
+        if not meta_url:
+            raise ExtractorError('No API host found')
+
+        uvid = video_data['data-uvid']
+        dtype = video_data.get('data-type')
+        stream_data = self._download_json(
+            '%s/api/%s/stream/%s' % (meta_url, 'show' if dtype == 'vod' else dtype, uvid),
+            uvid,
+            query={
+                'key': video_data.get('data-key'),
+                'platform': self._PLATFORM,
+            },
+            headers={
+                'Token': video_data.get('data-token'),
+                'Token-Expiry': video_data.get('data-expiry'),
+                'Uvid': uvid,
+            }, fatal=False)
+
+        stream_url = traverse_obj(stream_data, (
+            'response', 'stream', T(url_or_none)))
+        if not stream_url:
+            raise ExtractorError('No stream data/URL')
+
+        # now known to be a dict
+        stream_data = stream_data['response']
+        drm = stream_data.get('drm')
+        if drm:
+            self.report_drm(uvid)
+
+        formats = self._extract_m3u8_formats(
+            stream_url, uvid, ext='mp4', entry_protocol='m3u8_native',
+            fatal=False)
+        # exception if no formats
+        self._sort_formats(formats)
+
+        return {
+            'id': uvid,
+            'display_id': display_id,
+            'title': (traverse_obj(stream_data, ('title', T(txt_or_none)))
+                      or self._og_search_title(webpage, default=None)
+                      or display_id.replace('-', ' ').capitalize()),
+            'description': self._og_search_description(webpage, default=None),
+            'thumbnail': (traverse_obj(video_data, ('data-poster', T(url_or_none)))
+                          or self._og_search_thumbnail(webpage)),
+            'formats': formats,
+            'is_live': (dtype == 'live') or None,
+        }

From 40bd5c18153afe765caa6726302ee1dd8a9a2ce6 Mon Sep 17 00:00:00 2001
From: Aaron Tan <70739609+aaron-tan@users.noreply.github.com>
Date: Thu, 22 Feb 2024 23:54:07 +1100
Subject: [PATCH 731/860] [caffeine.tv] Add new extractor (#32514)

* Add CaffeineTVIE info extractor to support site caffeine.tv

---------

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/caffeine.py   | 79 ++++++++++++++++++++++++++++++
 youtube_dl/extractor/extractors.py |  1 +
 2 files changed, 80 insertions(+)
 create mode 100644 youtube_dl/extractor/caffeine.py

diff --git a/youtube_dl/extractor/caffeine.py b/youtube_dl/extractor/caffeine.py
new file mode 100644
index 000000000..bffedb9a7
--- /dev/null
+++ b/youtube_dl/extractor/caffeine.py
@@ -0,0 +1,79 @@
+# coding: utf-8
+from __future__ import unicode_literals
+
+from .common import InfoExtractor
+from ..utils import (
+    determine_ext,
+    int_or_none,
+    merge_dicts,
+    parse_iso8601,
+    T,
+    traverse_obj,
+    txt_or_none,
+    urljoin,
+)
+
+
+class CaffeineTVIE(InfoExtractor):
+    _VALID_URL = r'https?://(?:www\.)?caffeine\.tv/[^/]+/video/(?P<id>[0-9a-f-]+)'
+    _TESTS = [{
+        'url': 'https://www.caffeine.tv/TsuSurf/video/cffc0a00-e73f-11ec-8080-80017d29f26e',
+        'info_dict': {
+            'id': 'cffc0a00-e73f-11ec-8080-80017d29f26e',
+            'ext': 'mp4',
+            'title': 'GOOOOD MORNINNNNN #highlights',
+            'timestamp': 1654702180,
+            'upload_date': '20220608',
+            'uploader': 'TsuSurf',
+            'duration': 3145,
+            'age_limit': 17,
+        },
+        'params': {
+            'format': 'bestvideo',
+        },
+    }]
+
+    def _real_extract(self, url):
+        video_id = self._match_id(url)
+        json_data = self._download_json(
+            'https://api.caffeine.tv/social/public/activity/' + video_id,
+            video_id)
+        broadcast_info = traverse_obj(json_data, ('broadcast_info', T(dict))) or {}
+        title = broadcast_info['broadcast_title']
+        video_url = broadcast_info['video_url']
+
+        ext = determine_ext(video_url)
+        if ext == 'm3u8':
+            formats = self._extract_m3u8_formats(
+                video_url, video_id, 'mp4', entry_protocol='m3u8',
+                fatal=False)
+        else:
+            formats = [{'url': video_url}]
+        self._sort_formats(formats)
+
+        return merge_dicts({
+            'id': video_id,
+            'title': title,
+            'formats': formats,
+        }, traverse_obj(json_data, {
+            'uploader': ((None, 'user'), 'username'),
+        }, get_all=False), traverse_obj(json_data, {
+            'like_count': ('like_count', T(int_or_none)),
+            'view_count': ('view_count', T(int_or_none)),
+            'comment_count': ('comment_count', T(int_or_none)),
+            'tags': ('tags', Ellipsis, T(txt_or_none)),
+            'is_live': 'is_live',
+            'uploader': ('user', 'name'),
+        }), traverse_obj(broadcast_info, {
+            'duration': ('content_duration', T(int_or_none)),
+            'timestamp': ('broadcast_start_time', T(parse_iso8601)),
+            'thumbnail': ('preview_image_path', T(lambda u: urljoin(url, u))),
+            'age_limit': ('content_rating', T(lambda r: r and {
+                # assume Apple Store ratings [1]
+                # 1. https://en.wikipedia.org/wiki/Mobile_software_content_rating_system
+                'FOUR_PLUS': 0,
+                'NINE_PLUS': 9,
+                'TWELVE_PLUS': 12,
+                'SEVENTEEN_PLUS': 17,
+            }.get(r, 17))),
+        }))
diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index b6d0f42f5..29b655c94 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -159,6 +159,7 @@ from .businessinsider import BusinessInsiderIE
 from .buzzfeed import BuzzFeedIE
 from .byutv import BYUtvIE
 from .c56 import C56IE
+from .caffeine import CaffeineTVIE
 from .callin import CallinIE
 from .camdemy import (
     CamdemyIE,

From f0812d784836d18fd25ea32f9b5a0c9c6e92425b Mon Sep 17 00:00:00 2001
From: Hubert Hirtz <hubert@hirtz.pm>
Date: Mon, 4 Mar 2024 01:27:55 +0000
Subject: [PATCH 732/860] [utils] Handle user:pass in URLs (#28801)

* Handle user:pass in URLs

Fixes "nonnumeric port" errors when youtube-dl is given URLs with
usernames and passwords such as:

    http://username:password@example.com/myvideo.mp4

Refs:
- https://en.wikipedia.org/wiki/Basic_access_authentication
- https://tools.ietf.org/html/rfc1738#section-3.1
- https://docs.python.org/3.8/library/urllib.parse.html#urllib.parse.urlsplit

Fixes #18276 (point 4)
Fixes #20258
Fixes #26211 (see comment)

* Align code with yt-dlp

---------

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 test/test_utils.py  | 13 +++++++++++++
 youtube_dl/utils.py | 22 +++++++++++++++++++++-
 2 files changed, 34 insertions(+), 1 deletion(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 102420fcb..90d64b581 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -81,6 +81,7 @@ from youtube_dl.utils import (
     sanitize_filename,
     sanitize_path,
     sanitize_url,
+    sanitized_Request,
     shell_quote,
     smuggle_url,
     str_or_none,
@@ -255,6 +256,18 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(sanitize_url('https://foo.bar'), 'https://foo.bar')
         self.assertEqual(sanitize_url('foo bar'), 'foo bar')
 
+    def test_sanitized_Request(self):
+        self.assertFalse(sanitized_Request('http://foo.bar').has_header('Authorization'))
+        self.assertFalse(sanitized_Request('http://:foo.bar').has_header('Authorization'))
+        self.assertEqual(sanitized_Request('http://@foo.bar').get_header('Authorization'),
+                         'Basic Og==')
+        self.assertEqual(sanitized_Request('http://:pass@foo.bar').get_header('Authorization'),
+                         'Basic OnBhc3M=')
+        self.assertEqual(sanitized_Request('http://user:@foo.bar').get_header('Authorization'),
+                         'Basic dXNlcjo=')
+        self.assertEqual(sanitized_Request('http://user:pass@foo.bar').get_header('Authorization'),
+                         'Basic dXNlcjpwYXNz')
+
     def test_expand_path(self):
         def env(var):
             return '%{0}%'.format(var) if sys.platform == 'win32' else '${0}'.format(var)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 61b94d84c..c249e7168 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2182,8 +2182,28 @@ def sanitize_url(url):
     return url
 
 
+def extract_basic_auth(url):
+    parts = compat_urllib_parse.urlsplit(url)
+    if parts.username is None:
+        return url, None
+    url = compat_urllib_parse.urlunsplit(parts._replace(netloc=(
+        parts.hostname if parts.port is None
+        else '%s:%d' % (parts.hostname, parts.port))))
+    auth_payload = base64.b64encode(
+        ('%s:%s' % (parts.username, parts.password or '')).encode('utf-8'))
+    return url, 'Basic {0}'.format(auth_payload.decode('ascii'))
+
+
 def sanitized_Request(url, *args, **kwargs):
-    return compat_urllib_request.Request(escape_url(sanitize_url(url)), *args, **kwargs)
+    url, auth_header = extract_basic_auth(escape_url(sanitize_url(url)))
+    if auth_header is not None:
+        headers = args[1] if len(args) > 1 else kwargs.get('headers')
+        headers = headers or {}
+        headers['Authorization'] = auth_header
+        if len(args) <= 1 and kwargs.get('headers') is None:
+            kwargs['headers'] = headers
+            kwargs = compat_kwargs(kwargs)
+    return compat_urllib_request.Request(url, *args, **kwargs)
 
 
 def expand_path(s):

From acc383b9e3c2d454121c22570c901dd2c689dc26 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 4 Mar 2024 20:52:58 +0000
Subject: [PATCH 733/860] [utils] Let int_or_none() accept a base, like int()

---
 test/test_utils.py  | 3 +++
 youtube_dl/utils.py | 5 +++--
 2 files changed, 6 insertions(+), 2 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index 90d64b581..ca36909a8 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -525,11 +525,14 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(float_or_none(set()), None)
 
     def test_int_or_none(self):
+        self.assertEqual(int_or_none(42), 42)
         self.assertEqual(int_or_none('42'), 42)
         self.assertEqual(int_or_none(''), None)
         self.assertEqual(int_or_none(None), None)
         self.assertEqual(int_or_none([]), None)
         self.assertEqual(int_or_none(set()), None)
+        self.assertEqual(int_or_none('42', base=8), 34)
+        self.assertRaises(TypeError, int_or_none(42, base=8))
 
     def test_str_to_int(self):
         self.assertEqual(str_to_int('123,456'), 123456)
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index c249e7168..03c73dff3 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -3852,14 +3852,15 @@ class PUTRequest(compat_urllib_request.Request):
         return 'PUT'
 
 
-def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1):
+def int_or_none(v, scale=1, default=None, get_attr=None, invscale=1, base=None):
     if get_attr:
         if v is not None:
             v = getattr(v, get_attr, None)
     if v in (None, ''):
         return default
     try:
-        return int(v) * invscale // scale
+        # like int, raise if base is specified and v is not a string
+        return (int(v) if base is None else int(v, base=base)) * invscale // scale
     except (ValueError, TypeError, OverflowError):
         return default
 

From 7216fa2ac4706e099ea2ad9a04fe7bf4300bc745 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 21 Feb 2024 00:03:17 +0000
Subject: [PATCH 734/860] [InfoExtractor] Add `_search_json()` * uses the error
 diagnostic to truncate the JSON string * may be confused by non-C-Pythons

---
 youtube_dl/extractor/common.py | 55 ++++++++++++++++++++++++++++++++++
 1 file changed, 55 insertions(+)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 0e5dfd8fa..b5e95a318 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -25,6 +25,7 @@ from ..compat import (
     compat_getpass,
     compat_integer_types,
     compat_http_client,
+    compat_kwargs,
     compat_map as map,
     compat_open as open,
     compat_os_name,
@@ -1102,6 +1103,60 @@ class InfoExtractor(object):
             self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
             return None
 
+    def _search_json(self, start_pattern, string, name, video_id, **kwargs):
+        """Searches string for the JSON object specified by start_pattern"""
+
+        # self, start_pattern, string, name, video_id, *, end_pattern='',
+        # contains_pattern=r'{(?s:.+)}', fatal=True, default=NO_DEFAULT
+        # NB: end_pattern is only used to reduce the size of the initial match
+        end_pattern = kwargs.pop('end_pattern', '')
+        # (?:[\s\S]) simulates (?(s):.) (eg)
+        contains_pattern = kwargs.pop('contains_pattern', r'{[\s\S]+}')
+        fatal = kwargs.pop('fatal', True)
+        default = kwargs.pop('default', NO_DEFAULT)
+
+        if default is NO_DEFAULT:
+            default, has_default = {}, False
+        else:
+            fatal, has_default = False, True
+
+        json_string = self._search_regex(
+            r'(?:{0})\s*(?P<json>{1})\s*(?:{2})'.format(
+                start_pattern, contains_pattern, end_pattern),
+            string, name, group='json', fatal=fatal, default=None if has_default else NO_DEFAULT)
+        if not json_string:
+            return default
+
+        # yt-dlp has a special JSON parser that allows trailing text.
+        # Until that arrives here, the diagnostic from the exception
+        # raised by json.loads() is used to extract the wanted text.
+        # Either way, it's a problem if a transform_source() can't
+        # handle the trailing text.
+
+        # force an exception
+        kwargs['fatal'] = True
+
+        # self._downloader._format_err(name, self._downloader.Styles.EMPHASIS)
+        for _ in range(2):
+            try:
+                # return self._parse_json(json_string, video_id, ignore_extra=True, **kwargs)
+                transform_source = kwargs.pop('transform_source', None)
+                if transform_source:
+                    json_string = transform_source(json_string)
+                return self._parse_json(json_string, video_id, **compat_kwargs(kwargs))
+            except ExtractorError as e:
+                end = int_or_none(self._search_regex(r'\(char\s+(\d+)', error_to_compat_str(e), 'end', default=None))
+                if end is not None:
+                    json_string = json_string[:end]
+                    continue
+                msg = 'Unable to extract {0} - Failed to parse JSON'.format(name)
+                if fatal:
+                    raise ExtractorError(msg, cause=e.cause, video_id=video_id)
+                elif not has_default:
+                    self.report_warning(
+                        '{0}: {1}'.format(msg, error_to_compat_str(e)), video_id=video_id)
+            return default
+
     def _html_search_regex(self, pattern, string, name, default=NO_DEFAULT, fatal=True, flags=0, group=None):
         """
         Like _search_regex, but strips HTML tags and unescapes entities.

From f66372403fd9e1661199fea100ba2600fa9697b2 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 21 Feb 2024 00:09:48 +0000
Subject: [PATCH 735/860] [InfoExtractor] Rework and improve JWPlayer
 extraction * use traverse_obj() and _search_json() * support playlist
 `.load({**video1},{**video2}, ...)` * support transform_source=... for
 _extract_jwplayer_data()

---
 youtube_dl/extractor/common.py | 55 ++++++++++++++--------------------
 1 file changed, 22 insertions(+), 33 deletions(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b5e95a318..7fae9e57b 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -3021,25 +3021,22 @@ class InfoExtractor(object):
         return formats
 
     def _find_jwplayer_data(self, webpage, video_id=None, transform_source=js_to_json):
-        mobj = re.search(
-            r'''(?s)jwplayer\s*\(\s*(?P<q>'|")(?!(?P=q)).+(?P=q)\s*\)(?!</script>).*?\.\s*setup\s*\(\s*(?P<options>(?:\([^)]*\)|[^)])+)\s*\)''',
-            webpage)
-        if mobj:
-            try:
-                jwplayer_data = self._parse_json(mobj.group('options'),
-                                                 video_id=video_id,
-                                                 transform_source=transform_source)
-            except ExtractorError:
-                pass
-            else:
-                if isinstance(jwplayer_data, dict):
-                    return jwplayer_data
+        return self._search_json(
+            r'''(?<!-)\bjwplayer\s*\(\s*(?P<q>'|")(?!(?P=q)).+(?P=q)\s*\)(?:(?!</script>).)*?\.\s*(?:setup\s*\(|(?P<load>load)\s*\(\s*\[)''',
+            webpage, 'JWPlayer data', video_id,
+            # must be a {...} or sequence, ending
+            contains_pattern=r'\{[\s\S]*}(?(load)(?:\s*,\s*\{[\s\S]*})*)', end_pattern=r'(?(load)\]|\))',
+            transform_source=transform_source, default=None)
 
     def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
-        jwplayer_data = self._find_jwplayer_data(
-            webpage, video_id, transform_source=js_to_json)
-        return self._parse_jwplayer_data(
-            jwplayer_data, video_id, *args, **kwargs)
+
+        # allow passing `transform_source` through to _find_jwplayer_data()
+        transform_source = kwargs.pop('transform_source', None)
+        kwfind = compat_kwargs({'transform_source': transform_source}) if transform_source else {}
+
+        jwplayer_data = self._find_jwplayer_data(webpage, video_id, **kwfind)
+
+        return self._parse_jwplayer_data(jwplayer_data, video_id, *args, **kwargs)
 
     def _parse_jwplayer_data(self, jwplayer_data, video_id=None, require_title=True,
                              m3u8_id=None, mpd_id=None, rtmp_params=None, base_url=None):
@@ -3073,22 +3070,14 @@ class InfoExtractor(object):
                 mpd_id=mpd_id, rtmp_params=rtmp_params, base_url=base_url)
 
             subtitles = {}
-            tracks = video_data.get('tracks')
-            if tracks and isinstance(tracks, list):
-                for track in tracks:
-                    if not isinstance(track, dict):
-                        continue
-                    track_kind = track.get('kind')
-                    if not track_kind or not isinstance(track_kind, compat_str):
-                        continue
-                    if track_kind.lower() not in ('captions', 'subtitles'):
-                        continue
-                    track_url = urljoin(base_url, track.get('file'))
-                    if not track_url:
-                        continue
-                    subtitles.setdefault(track.get('label') or 'en', []).append({
-                        'url': self._proto_relative_url(track_url)
-                    })
+            for track in traverse_obj(video_data, (
+                    'tracks', lambda _, t: t.get('kind').lower() in ('captions', 'subtitles'))):
+                track_url = urljoin(base_url, track.get('file'))
+                if not track_url:
+                    continue
+                subtitles.setdefault(track.get('label') or 'en', []).append({
+                    'url': self._proto_relative_url(track_url)
+                })
 
             entry = {
                 'id': this_video_id,

From f7b30e3f73f56aa4765212cd04eb48743e03dfcd Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 21 Feb 2024 00:18:17 +0000
Subject: [PATCH 736/860] [XFileShare] Update extractor for 2024 * simplify
 aa_decode() * review and update supported sites and tests * in above, include
 FileMoon.sx, and remove separate module * incorporate changes from yt-dlp *
 allow for decoding multiple scripts (eg, FileMoon) * use new JWPlayer
 extraction

---
 youtube_dl/extractor/extractors.py |   1 -
 youtube_dl/extractor/filemoon.py   |  43 -----
 youtube_dl/extractor/xfileshare.py | 290 +++++++++++++++++++----------
 3 files changed, 190 insertions(+), 144 deletions(-)
 delete mode 100644 youtube_dl/extractor/filemoon.py

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 29b655c94..a56a7c52f 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -383,7 +383,6 @@ from .fc2 import (
     FC2EmbedIE,
 )
 from .fczenit import FczenitIE
-from .filemoon import FileMoonIE
 from .fifa import FifaIE
 from .filmon import (
     FilmOnIE,
diff --git a/youtube_dl/extractor/filemoon.py b/youtube_dl/extractor/filemoon.py
deleted file mode 100644
index 654df9b69..000000000
--- a/youtube_dl/extractor/filemoon.py
+++ /dev/null
@@ -1,43 +0,0 @@
-# coding: utf-8
-from __future__ import unicode_literals
-
-import re
-
-from .common import InfoExtractor
-from ..utils import (
-    decode_packed_codes,
-    js_to_json,
-)
-
-
-class FileMoonIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?filemoon\.sx/./(?P<id>\w+)'
-    _TEST = {
-        'url': 'https://filemoon.sx/e/dw40rxrzruqz',
-        'md5': '5a713742f57ac4aef29b74733e8dda01',
-        'info_dict': {
-            'id': 'dw40rxrzruqz',
-            'title': 'dw40rxrzruqz',
-            'ext': 'mp4'
-        }
-    }
-
-    def _real_extract(self, url):
-        video_id = self._match_id(url)
-
-        webpage = self._download_webpage(url, video_id)
-        matches = re.findall(r'(?s)(eval.*?)</script>', webpage)
-        packed = matches[-1]
-        unpacked = decode_packed_codes(packed)
-        jwplayer_sources = self._parse_json(
-            self._search_regex(
-                r'(?s)player\s*\.\s*setup\s*\(\s*\{\s*sources\s*:\s*(.*?])', unpacked, 'jwplayer sources'),
-            video_id, transform_source=js_to_json)
-
-        formats = self._parse_jwplayer_formats(jwplayer_sources, video_id)
-
-        return {
-            'id': video_id,
-            'title': self._generic_title(url) or video_id,
-            'formats': formats
-        }
diff --git a/youtube_dl/extractor/xfileshare.py b/youtube_dl/extractor/xfileshare.py
index df9efa9fa..4dc3032e7 100644
--- a/youtube_dl/extractor/xfileshare.py
+++ b/youtube_dl/extractor/xfileshare.py
@@ -4,20 +4,28 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_chr
+from ..compat import (
+    compat_chr,
+    compat_zip as zip,
+)
 from ..utils import (
+    clean_html,
     decode_packed_codes,
     determine_ext,
     ExtractorError,
+    get_element_by_id,
     int_or_none,
-    js_to_json,
+    merge_dicts,
+    T,
+    traverse_obj,
+    url_or_none,
     urlencode_postdata,
 )
 
 
 # based on openload_decode from 2bfeee69b976fe049761dd3012e30b637ee05a58
 def aa_decode(aa_code):
-    symbol_table = [
+    symbol_table = (
         ('7', '((ﾟｰﾟ) + (o^_^o))'),
         ('6', '((o^_^o) +(o^_^o))'),
         ('5', '((ﾟｰﾟ) + (ﾟΘﾟ))'),
@@ -26,84 +34,180 @@ def aa_decode(aa_code):
         ('3', '(o^_^o)'),
         ('1', '(ﾟΘﾟ)'),
         ('0', '(c^_^o)'),
-    ]
+        ('+', ''),
+    )
     delim = '(ﾟДﾟ)[ﾟεﾟ]+'
-    ret = ''
-    for aa_char in aa_code.split(delim):
+
+    def chr_from_code(c):
         for val, pat in symbol_table:
-            aa_char = aa_char.replace(pat, val)
-        aa_char = aa_char.replace('+ ', '')
-        m = re.match(r'^\d+', aa_char)
-        if m:
-            ret += compat_chr(int(m.group(0), 8))
+            c = c.replace(pat, val)
+        if c.startswith(('u', 'U')):
+            base = 16
+            c = c[1:]
         else:
-            m = re.match(r'^u([\da-f]+)', aa_char)
-            if m:
-                ret += compat_chr(int(m.group(1), 16))
-    return ret
+            base = 10
+        c = int_or_none(c, base=base)
+        return '' if c is None else compat_chr(c)
+
+    return ''.join(
+        chr_from_code(aa_char)
+        for aa_char in aa_code.split(delim))
 
 
 class XFileShareIE(InfoExtractor):
     _SITES = (
-        (r'aparat\.cam', 'Aparat'),
-        (r'clipwatching\.com', 'ClipWatching'),
-        (r'gounlimited\.to', 'GoUnlimited'),
-        (r'govid\.me', 'GoVid'),
-        (r'holavid\.com', 'HolaVid'),
-        (r'streamty\.com', 'Streamty'),
-        (r'thevideobee\.to', 'TheVideoBee'),
-        (r'uqload\.com', 'Uqload'),
-        (r'vidbom\.com', 'VidBom'),
-        (r'vidlo\.us', 'vidlo'),
-        (r'vidlocker\.xyz', 'VidLocker'),
-        (r'vidshare\.tv', 'VidShare'),
-        (r'vup\.to', 'VUp'),
+        # status check 2024-02: site availability, G site: search
+        (r'aparat\.cam', 'Aparat'),  # Cloudflare says host error 522, apparently changed to wolfstreeam.tv
+        (r'filemoon\.sx/.', 'FileMoon'),
+        (r'gounlimited\.to', 'GoUnlimited'),  # no media pages listed
+        (r'govid\.me', 'GoVid'),  # no media pages listed
+        (r'highstream\.tv', 'HighStream'),  # clipwatching.com redirects here
+        (r'holavid\.com', 'HolaVid'),  # Cloudflare says host error 522
+        # (r'streamty\.com', 'Streamty'),  # no media pages listed, connection timeout
+        # (r'thevideobee\.to', 'TheVideoBee'),  # no pages listed, refuses connection
+        (r'uqload\.to', 'Uqload'),  # .com, .co redirect here
+        (r'(?:vedbam\.xyz|vadbam.net)', 'V?dB?m'),  # vidbom.com redirects here, but no valid media pages listed
+        (r'vidlo\.us', 'vidlo'),  # no valid media pages listed
+        (r'vidlocker\.xyz', 'VidLocker'),  # no media pages listed
+        (r'(?:w\d\.)?viidshar\.com', 'VidShare'),  # vidshare.tv redirects here
+        # (r'vup\.to', 'VUp'),  # domain not found
         (r'wolfstream\.tv', 'WolfStream'),
-        (r'xvideosharing\.com', 'XVideoSharing'),
+        (r'xvideosharing\.com', 'XVideoSharing'),  # just started showing 'maintenance mode'
     )
 
-    IE_DESC = 'XFileShare based sites: %s' % ', '.join(list(zip(*_SITES))[1])
+    IE_DESC = 'XFileShare-based sites: %s' % ', '.join(list(zip(*_SITES))[1])
     _VALID_URL = (r'https?://(?:www\.)?(?P<host>%s)/(?:embed-)?(?P<id>[0-9a-zA-Z]+)'
                   % '|'.join(site for site in list(zip(*_SITES))[0]))
+    _EMBED_REGEX = [r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:%s)/embed-[0-9a-zA-Z]+.*?)\1' % '|'.join(site for site in list(zip(*_SITES))[0])]
 
     _FILE_NOT_FOUND_REGEXES = (
         r'>(?:404 - )?File Not Found<',
         r'>The file was removed by administrator<',
     )
+    _TITLE_REGEXES = (
+        r'style="z-index: [0-9]+;">([^<]+)</span>',
+        r'<td nowrap>([^<]+)</td>',
+        r'h4-fine[^>]*>([^<]+)<',
+        r'>Watch (.+)[ <]',
+        r'<h2 class="video-page-head">([^<]+)</h2>',
+        r'<h2 style="[^"]*color:#403f3d[^"]*"[^>]*>([^<]+)<',  # streamin.to (dead)
+        r'title\s*:\s*"([^"]+)"',  # govid.me
+    )
+    _SOURCE_URL_REGEXES = (
+        r'(?:file|src)\s*:\s*(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1',
+        r'file_link\s*=\s*(["\'])(?P<url>http(?:(?!\1).)+)\1',
+        r'addVariable\((\\?["\'])file\1\s*,\s*(\\?["\'])(?P<url>http(?:(?!\2).)+)\2\)',
+        r'<embed[^>]+src=(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1',
+    )
+    _THUMBNAIL_REGEXES = (
+        r'<video[^>]+poster="([^"]+)"',
+        r'(?:image|poster)\s*:\s*["\'](http[^"\']+)["\'],',
+    )
 
     _TESTS = [{
-        'url': 'http://xvideosharing.com/fq65f94nd2ve',
-        'md5': '4181f63957e8fe90ac836fa58dc3c8a6',
+        'note': 'link in `sources`',
+        'url': 'https://uqload.to/dcsu06gdb45o',
+        'md5': '7f8db187b254379440bf4fcad094ae86',
         'info_dict': {
-            'id': 'fq65f94nd2ve',
+            'id': 'dcsu06gdb45o',
             'ext': 'mp4',
-            'title': 'sample',
-            'thumbnail': r're:http://.*\.jpg',
+            'title': 'f2e31015957e74c8c8427982e161c3fc mp4',
+            'thumbnail': r're:https://.*\.jpg'
+        },
+        'params': {
+            'nocheckcertificate': True,
+        },
+        'expected_warnings': ['Unable to extract JWPlayer data'],
+    }, {
+        'note': 'link in decoded `sources`',
+        'url': 'https://xvideosharing.com/1tlg6agrrdgc',
+        'md5': '2608ce41932c1657ae56258a64e647d9',
+        'info_dict': {
+            'id': '1tlg6agrrdgc',
+            'ext': 'mp4',
+            'title': '0121',
+            'thumbnail': r're:https?://.*\.jpg',
+        },
+        'skip': 'This server is in maintenance mode.',
+    }, {
+        'note': 'JWPlayer link in un-p,a,c,k,e,d JS',
+        'url': 'https://filemoon.sx/e/dw40rxrzruqz',
+        'md5': '5a713742f57ac4aef29b74733e8dda01',
+        'info_dict': {
+            'id': 'dw40rxrzruqz',
+            'title': 'dw40rxrzruqz',
+            'ext': 'mp4'
+        },
+    }, {
+        'note': 'JWPlayer link in un-p,a,c,k,e,d JS',
+        'url': 'https://vadbam.net/6lnbkci96wly.html',
+        'md5': 'a1616800076177e2ac769203957c54bc',
+        'info_dict': {
+            'id': '6lnbkci96wly',
+            'title': 'Heart Crime S01 E03 weciima autos',
+            'ext': 'mp4'
+        },
+    }, {
+        'note': 'JWPlayer link in clear',
+        'url': 'https://w1.viidshar.com/nnibe0xf0h79.html',
+        'md5': 'f0a580ce9df06cc61b4a5c979d672367',
+        'info_dict': {
+            'id': 'nnibe0xf0h79',
+            'title': 'JaGa 68ar',
+            'ext': 'mp4'
+        },
+        'params': {
+            'skip_download': 'ffmpeg',
+        },
+        'expected_warnings': ['hlsnative has detected features it does not support'],
+    }, {
+        'note': 'JWPlayer link in clear',
+        'url': 'https://wolfstream.tv/a3drtehyrg52.html',
+        'md5': '1901d86a79c5e0c6a51bdc9a4cfd3769',
+        'info_dict': {
+            'id': 'a3drtehyrg52',
+            'title': 'NFL 2023 W04 DET@GB',
+            'ext': 'mp4'
         },
     }, {
         'url': 'https://aparat.cam/n4d6dh0wvlpr',
         'only_matching': True,
     }, {
-        'url': 'https://wolfstream.tv/nthme29v9u2x',
+        'url': 'https://uqload.to/ug5somm0ctnk.html',
+        'only_matching': True,
+    }, {
+        'url': 'https://highstream.tv/2owiyz3sjoux',
+        'only_matching': True,
+    }, {
+        'url': 'https://vedbam.xyz/6lnbkci96wly.html',
         'only_matching': True,
     }]
 
-    @staticmethod
-    def _extract_urls(webpage):
-        return [
-            mobj.group('url')
-            for mobj in re.finditer(
-                r'<iframe\b[^>]+\bsrc=(["\'])(?P<url>(?:https?:)?//(?:%s)/embed-[0-9a-zA-Z]+.*?)\1'
-                % '|'.join(site for site in list(zip(*XFileShareIE._SITES))[0]),
-                webpage)]
+    @classmethod
+    def _extract_urls(cls, webpage):
+
+        def yield_urls():
+            for regex in cls._EMBED_REGEX:
+                for mobj in re.finditer(regex, webpage):
+                    yield mobj.group('url')
+
+        return list(yield_urls())
 
     def _real_extract(self, url):
-        host, video_id = re.match(self._VALID_URL, url).groups()
+        host, video_id = self._match_valid_url(url).group('host', 'id')
 
-        url = 'https://%s/' % host + ('embed-%s.html' % video_id if host in ('govid.me', 'vidlo.us') else video_id)
+        url = 'https://%s/%s' % (
+            host,
+            'embed-%s.html' % video_id if host in ('govid.me', 'vidlo.us') else video_id)
         webpage = self._download_webpage(url, video_id)
-
-        if any(re.search(p, webpage) for p in self._FILE_NOT_FOUND_REGEXES):
+        container_div = get_element_by_id('container', webpage) or webpage
+        if self._search_regex(
+                r'>This server is in maintenance mode\.', container_div,
+                'maint error', group=0, default=None):
+            raise ExtractorError(clean_html(container_div), expected=True)
+        if self._search_regex(
+                self._FILE_NOT_FOUND_REGEXES, container_div,
+                'missing video error', group=0, default=None):
             raise ExtractorError('Video %s does not exist' % video_id, expected=True)
 
         fields = self._hidden_inputs(webpage)
@@ -122,59 +226,43 @@ class XFileShareIE(InfoExtractor):
                     'Content-type': 'application/x-www-form-urlencoded',
                 })
 
-        title = (self._search_regex(
-            (r'style="z-index: [0-9]+;">([^<]+)</span>',
-             r'<td nowrap>([^<]+)</td>',
-             r'h4-fine[^>]*>([^<]+)<',
-             r'>Watch (.+)[ <]',
-             r'<h2 class="video-page-head">([^<]+)</h2>',
-             r'<h2 style="[^"]*color:#403f3d[^"]*"[^>]*>([^<]+)<',  # streamin.to
-             r'title\s*:\s*"([^"]+)"'),  # govid.me
-            webpage, 'title', default=None) or self._og_search_title(
-            webpage, default=None) or video_id).strip()
+        title = (
+            self._search_regex(self._TITLE_REGEXES, webpage, 'title', default=None)
+            or self._og_search_title(webpage, default=None)
+            or video_id).strip()
 
-        for regex, func in (
-                (r'(eval\(function\(p,a,c,k,e,d\){.+)', decode_packed_codes),
-                (r'(ﾟ.+)', aa_decode)):
-            obf_code = self._search_regex(regex, webpage, 'obfuscated code', default=None)
-            if obf_code:
-                webpage = webpage.replace(obf_code, func(obf_code))
+        obf_code = True
+        while obf_code:
+            for regex, func in (
+                    (r'(?s)(?<!-)\b(eval\(function\(p,a,c,k,e,d\)\{(?:(?!</script>).)+\)\))',
+                     decode_packed_codes),
+                    (r'(ﾟ.+)', aa_decode)):
+                obf_code = self._search_regex(regex, webpage, 'obfuscated code', default=None)
+                if obf_code:
+                    webpage = webpage.replace(obf_code, func(obf_code))
+                    break
 
-        formats = []
+        jwplayer_data = self._find_jwplayer_data(
+            webpage.replace(r'\'', '\''), video_id)
+        result = self._parse_jwplayer_data(
+            jwplayer_data, video_id, require_title=False,
+            m3u8_id='hls', mpd_id='dash')
 
-        jwplayer_data = self._search_regex(
-            [
-                r'jwplayer\("[^"]+"\)\.load\(\[({.+?})\]\);',
-                r'jwplayer\("[^"]+"\)\.setup\(({.+?})\);',
-            ], webpage,
-            'jwplayer data', default=None)
-        if jwplayer_data:
-            jwplayer_data = self._parse_json(
-                jwplayer_data.replace(r"\'", "'"), video_id, js_to_json)
+        if not traverse_obj(result, 'formats'):
             if jwplayer_data:
-                formats = self._parse_jwplayer_data(
-                    jwplayer_data, video_id, False,
-                    m3u8_id='hls', mpd_id='dash')['formats']
-
-        if not formats:
-            urls = []
-            for regex in (
-                    r'(?:file|src)\s*:\s*(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1',
-                    r'file_link\s*=\s*(["\'])(?P<url>http(?:(?!\1).)+)\1',
-                    r'addVariable\((\\?["\'])file\1\s*,\s*(\\?["\'])(?P<url>http(?:(?!\2).)+)\2\)',
-                    r'<embed[^>]+src=(["\'])(?P<url>http(?:(?!\1).)+\.(?:m3u8|mp4|flv)(?:(?!\1).)*)\1'):
+                self.report_warning(
+                    'Failed to extract JWPlayer formats', video_id=video_id)
+            urls = set()
+            for regex in self._SOURCE_URL_REGEXES:
                 for mobj in re.finditer(regex, webpage):
-                    video_url = mobj.group('url')
-                    if video_url not in urls:
-                        urls.append(video_url)
+                    urls.add(mobj.group('url'))
 
             sources = self._search_regex(
                 r'sources\s*:\s*(\[(?!{)[^\]]+\])', webpage, 'sources', default=None)
-            if sources:
-                urls.extend(self._parse_json(sources, video_id))
+            urls.update(traverse_obj(sources, (T(lambda s: self._parse_json(s, video_id)), Ellipsis)))
 
             formats = []
-            for video_url in urls:
+            for video_url in traverse_obj(urls, (Ellipsis, T(url_or_none))):
                 if determine_ext(video_url) == 'm3u8':
                     formats.extend(self._extract_m3u8_formats(
                         video_url, video_id, 'mp4',
@@ -185,17 +273,19 @@ class XFileShareIE(InfoExtractor):
                         'url': video_url,
                         'format_id': 'sd',
                     })
-        self._sort_formats(formats)
+            result = {'formats': formats}
+
+        self._sort_formats(result['formats'])
 
         thumbnail = self._search_regex(
-            [
-                r'<video[^>]+poster="([^"]+)"',
-                r'(?:image|poster)\s*:\s*["\'](http[^"\']+)["\'],',
-            ], webpage, 'thumbnail', default=None)
+            self._THUMBNAIL_REGEXES, webpage, 'thumbnail', default=None)
 
-        return {
+        if not (title or result.get('title')):
+            title = self._generic_title(url) or video_id
+
+        return merge_dicts(result, {
             'id': video_id,
-            'title': title,
+            'title': title or None,
             'thumbnail': thumbnail,
-            'formats': formats,
-        }
+            'http_headers': {'Referer': url}
+        })

From aef24d97e9c50cd9db06349b2b25c7f623baf932 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 29 Feb 2024 14:12:37 +0000
Subject: [PATCH 737/860] [Videa] Align with yt-dlp

---
 youtube_dl/extractor/videa.py | 39 +++++++++++++++++++++++------------
 1 file changed, 26 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/videa.py b/youtube_dl/extractor/videa.py
index 4589e78a1..0689764a5 100644
--- a/youtube_dl/extractor/videa.py
+++ b/youtube_dl/extractor/videa.py
@@ -6,22 +6,31 @@ import re
 import string
 
 from .common import InfoExtractor
+from ..compat import (
+    compat_b64decode,
+    compat_ord,
+    compat_struct_pack,
+)
 from ..utils import (
     ExtractorError,
     int_or_none,
     mimetype2ext,
     parse_codecs,
+    parse_qs,
     update_url_query,
     urljoin,
     xpath_element,
     xpath_text,
 )
-from ..compat import (
-    compat_b64decode,
-    compat_ord,
-    compat_struct_pack,
-    compat_urlparse,
-)
+
+
+def compat_random_choices(population, *args, **kwargs):
+    # weights=None, *, cum_weights=None, k=1
+    # limited implementation needed here
+    weights = args[0] if args else kwargs.get('weights')
+    assert all(w is None for w in (weights, kwargs.get('cum_weights')))
+    k = kwargs.get('k', 1)
+    return ''.join(random.choice(population) for _ in range(k))
 
 
 class VideaIE(InfoExtractor):
@@ -35,6 +44,7 @@ class VideaIE(InfoExtractor):
                         )
                         (?P<id>[^?#&]+)
                     '''
+    _EMBED_REGEX = [r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//videa\.hu/player\?.*?\bv=.+?)\1']
     _TESTS = [{
         'url': 'http://videa.hu/videok/allatok/az-orult-kigyasz-285-kigyot-kigyo-8YfIAjxwWGwT8HVQ',
         'md5': '97a7af41faeaffd9f1fc864a7c7e7603',
@@ -80,11 +90,14 @@ class VideaIE(InfoExtractor):
     }]
     _STATIC_SECRET = 'xHb0ZvME5q8CBcoQi6AngerDu3FGO9fkUlwPmLVY_RTzj2hJIS4NasXWKy1td7p'
 
-    @staticmethod
-    def _extract_urls(webpage):
-        return [url for _, url in re.findall(
-            r'<iframe[^>]+src=(["\'])(?P<url>(?:https?:)?//videa\.hu/player\?.*?\bv=.+?)\1',
-            webpage)]
+    @classmethod
+    def _extract_urls(cls, webpage):
+        def yield_urls():
+            for pattern in cls._EMBED_REGEX:
+                for m in re.finditer(pattern, webpage):
+                    yield m.group('url')
+
+        return list(yield_urls())
 
     @staticmethod
     def rc4(cipher_text, key):
@@ -130,8 +143,8 @@ class VideaIE(InfoExtractor):
         for i in range(0, 32):
             result += s[i - (self._STATIC_SECRET.index(l[i]) - 31)]
 
-        query = compat_urlparse.parse_qs(compat_urlparse.urlparse(player_url).query)
-        random_seed = ''.join(random.choice(string.ascii_letters + string.digits) for _ in range(8))
+        query = parse_qs(player_url)
+        random_seed = ''.join(compat_random_choices(string.ascii_letters + string.digits, k=8))
         query['_s'] = random_seed
         query['_t'] = result[:16]
 

From 820fae3b3a8587a6f57afbe803b4f91de7d4e086 Mon Sep 17 00:00:00 2001
From: hatsomatt <143712404+hatsomatt@users.noreply.github.com>
Date: Sat, 16 Sep 2023 16:02:37 +0200
Subject: [PATCH 738/860] [Videa] Fix extraction * update API URL * from
 https://github.com/yt-dlp/yt-dlp/pull/8003 * thanks to the authors!

Closes yt-dlp/7427
Authored by: hatsomatt, aky-01
---
 youtube_dl/extractor/videa.py | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/videa.py b/youtube_dl/extractor/videa.py
index 0689764a5..194b4b011 100644
--- a/youtube_dl/extractor/videa.py
+++ b/youtube_dl/extractor/videa.py
@@ -54,6 +54,7 @@ class VideaIE(InfoExtractor):
             'title': 'Az őrült kígyász 285 kígyót enged szabadon',
             'thumbnail': r're:^https?://.*',
             'duration': 21,
+            'age_limit': 0,
         },
     }, {
         'url': 'http://videa.hu/videok/origo/jarmuvek/supercars-elozes-jAHDWfWSJH5XuFhH',
@@ -64,6 +65,7 @@ class VideaIE(InfoExtractor):
             'title': 'Supercars előzés',
             'thumbnail': r're:^https?://.*',
             'duration': 64,
+            'age_limit': 0,
         },
     }, {
         'url': 'http://videa.hu/player?v=8YfIAjxwWGwT8HVQ',
@@ -74,6 +76,7 @@ class VideaIE(InfoExtractor):
             'title': 'Az őrült kígyász 285 kígyót enged szabadon',
             'thumbnail': r're:^https?://.*',
             'duration': 21,
+            'age_limit': 0,
         },
     }, {
         'url': 'http://videa.hu/player/v/8YfIAjxwWGwT8HVQ?autoplay=1',
@@ -149,7 +152,7 @@ class VideaIE(InfoExtractor):
         query['_t'] = result[:16]
 
         b64_info, handle = self._download_webpage_handle(
-            'http://videa.hu/videaplayer_get_xml.php', video_id, query=query)
+            'http://videa.hu/player/xml', video_id, query=query)
         if b64_info.startswith('<?xml'):
             info = self._parse_xml(b64_info, video_id)
         else:

From a96a45b2cdcfa5f20ae4264bed268cce93fbc521 Mon Sep 17 00:00:00 2001
From: Zizheng Guo <19143357+gzz2000@users.noreply.github.com>
Date: Tue, 12 Mar 2024 19:44:13 +0800
Subject: [PATCH 739/860] [Vimeo] Improve `config` extraction (#32742)

* update for more robust json parsing
---
 youtube_dl/extractor/vimeo.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/vimeo.py b/youtube_dl/extractor/vimeo.py
index 8e1a805f6..47ec0a9b4 100644
--- a/youtube_dl/extractor/vimeo.py
+++ b/youtube_dl/extractor/vimeo.py
@@ -673,8 +673,8 @@ class VimeoIE(VimeoBaseInfoExtractor):
             raise
 
         if '//player.vimeo.com/video/' in url:
-            config = self._parse_json(self._search_regex(
-                r'(?s)\b(?:playerC|c)onfig\s*=\s*({.+?})\s*[;\n]', webpage, 'info section'), video_id)
+            config = self._search_json(
+                r'\b(?:playerC|c)onfig\s*=', webpage, 'info section', video_id)
             if config.get('view') == 4:
                 config = self._verify_player_video_password(
                     redirect_url, video_id, headers)

From 71211e7db7243377f862dfdea9a9c3a511df66c2 Mon Sep 17 00:00:00 2001
From: gy-chen <gychen@gyhost.icu>
Date: Sat, 23 Mar 2024 23:30:13 +0800
Subject: [PATCH 740/860] [Youtube] Fix unwanted private method __ie_msg in
 f8b0135850

Fixes `AttributeError no attribute '_YoutubeIE__ie_msg'` if unable to decode n-parameter
---
 youtube_dl/extractor/youtube.py | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index db840fc45..90c16e172 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1647,10 +1647,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         except JSInterpreter.Exception as e:
             self.report_warning(
                 '%s (%s %s)' % (
-                    self.__ie_msg(
-                        'Unable to decode n-parameter: download likely to be throttled'),
+                    'Unable to decode n-parameter: download likely to be throttled',
                     error_to_compat_str(e),
-                    traceback.format_exc()))
+                    traceback.format_exc()),
+                video_id=video_id)
             return
 
         self.write_debug('Decrypted nsig {0} => {1}'.format(n, ret))

From 182f63e82a390e138f4a133d3ccb9c838222b02d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 1 Mar 2024 15:25:44 +0000
Subject: [PATCH 741/860] [compat] Add compat_contextlib_suppress

with compat_contextlib_suppress(*Exceptions):
    # code that fails silently for any of Exceptions
---
 youtube_dl/compat.py | 19 +++++++++++++++++++
 1 file changed, 19 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 818ccebd0..637f0d82c 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2943,6 +2943,24 @@ else:
     compat_socket_create_connection = socket.create_connection
 
 
+try:
+    from contextlib import suppress as compat_contextlib_suppress
+except ImportError:
+    class compat_contextlib_suppress(object):
+        _exceptions = None
+
+        def __init__(self, *exceptions):
+            super(compat_contextlib_suppress, self).__init__()
+            # TODO: [Base]ExceptionGroup (3.12+)
+            self._exceptions = exceptions
+
+        def __enter__(self):
+            return self
+
+        def __exit__(self, exc_type, exc_val, exc_tb):
+            return exc_val is not None and isinstance(exc_val, self._exceptions or tuple())
+
+
 # Fix https://github.com/ytdl-org/youtube-dl/issues/4223
 # See http://bugs.python.org/issue9161 for what is broken
 def workaround_optparse_bug9161():
@@ -3263,6 +3281,7 @@ __all__ = [
     'compat_http_cookiejar_Cookie',
     'compat_http_cookies',
     'compat_http_cookies_SimpleCookie',
+    'compat_contextlib_suppress',
     'compat_ctypes_WINFUNCTYPE',
     'compat_etree_fromstring',
     'compat_filter',

From 19dc10b986eeda47975a0e77e84df20ad3b59062 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 1 Mar 2024 19:53:58 +0000
Subject: [PATCH 742/860] [utils] Apply compat_contextlib_suppress

---
 youtube_dl/utils.py | 66 +++++++++++++++++----------------------------
 1 file changed, 24 insertions(+), 42 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 03c73dff3..083446342 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -45,6 +45,7 @@ from .compat import (
     compat_casefold,
     compat_chr,
     compat_collections_abc,
+    compat_contextlib_suppress,
     compat_cookiejar,
     compat_ctypes_WINFUNCTYPE,
     compat_datetime_timedelta_total_seconds,
@@ -1855,25 +1856,18 @@ def write_json_file(obj, fn):
     try:
         with tf:
             json.dump(obj, tf)
-        if sys.platform == 'win32':
-            # Need to remove existing file on Windows, else os.rename raises
-            # WindowsError or FileExistsError.
-            try:
+        with compat_contextlib_suppress(OSError):
+            if sys.platform == 'win32':
+                # Need to remove existing file on Windows, else os.rename raises
+                # WindowsError or FileExistsError.
                 os.unlink(fn)
-            except OSError:
-                pass
-        try:
             mask = os.umask(0)
             os.umask(mask)
             os.chmod(tf.name, 0o666 & ~mask)
-        except OSError:
-            pass
         os.rename(tf.name, fn)
     except Exception:
-        try:
+        with compat_contextlib_suppress(OSError):
             os.remove(tf.name)
-        except OSError:
-            pass
         raise
 
 
@@ -2033,14 +2027,13 @@ def extract_attributes(html_element):
     NB HTMLParser is stricter in Python 2.6 & 3.2 than in later versions,
     but the cases in the unit test will work for all of 2.6, 2.7, 3.2-3.5.
     """
-    parser = HTMLAttributeParser()
-    try:
-        parser.feed(html_element)
-        parser.close()
-    # Older Python may throw HTMLParseError in case of malformed HTML
-    except compat_HTMLParseError:
-        pass
-    return parser.attrs
+    ret = None
+    # Older Python may throw HTMLParseError in case of malformed HTML (and on .close()!)
+    with compat_contextlib_suppress(compat_HTMLParseError):
+        with contextlib.closing(HTMLAttributeParser()) as parser:
+            parser.feed(html_element)
+            ret = parser.attrs
+    return ret or {}
 
 
 def clean_html(html):
@@ -2241,7 +2234,8 @@ def _htmlentity_transform(entity_with_semicolon):
             numstr = '0%s' % numstr
         else:
             base = 10
-        # See https://github.com/ytdl-org/youtube-dl/issues/7518
+        # See https://github.com/ytdl-org/youtube-dl/issues/7518\
+        # Also, weirdly, compat_contextlib_suppress fails here in 2.6
         try:
             return compat_chr(int(numstr, base))
         except ValueError:
@@ -2348,11 +2342,9 @@ def make_HTTPS_handler(params, **kwargs):
         # Some servers may (wrongly) reject requests if ALPN extension is not sent. See:
         # https://github.com/python/cpython/issues/85140
         # https://github.com/yt-dlp/yt-dlp/issues/3878
-        try:
+        with compat_contextlib_suppress(AttributeError, NotImplementedError):
+            # fails for Python < 2.7.10, not ssl.HAS_ALPN
             ctx.set_alpn_protocols(ALPN_PROTOCOLS)
-        except (AttributeError, NotImplementedError):
-            # Python < 2.7.10, not ssl.HAS_ALPN
-            pass
 
     opts_no_check_certificate = params.get('nocheckcertificate', False)
     if hasattr(ssl, 'create_default_context'):  # Python >= 3.4 or 2.7.9
@@ -2362,12 +2354,10 @@ def make_HTTPS_handler(params, **kwargs):
             context.check_hostname = False
             context.verify_mode = ssl.CERT_NONE
 
-        try:
+        with compat_contextlib_suppress(TypeError):
+            # Fails with Python 2.7.8 (create_default_context present
+            # but HTTPSHandler has no context=)
             return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
-        except TypeError:
-            # Python 2.7.8
-            # (create_default_context present but HTTPSHandler has no context=)
-            pass
 
     if sys.version_info < (3, 2):
         return YoutubeDLHTTPSHandler(params, **kwargs)
@@ -3176,12 +3166,10 @@ def parse_iso8601(date_str, delimiter='T', timezone=None):
     if timezone is None:
         timezone, date_str = extract_timezone(date_str)
 
-    try:
+    with compat_contextlib_suppress(ValueError):
         date_format = '%Y-%m-%d{0}%H:%M:%S'.format(delimiter)
         dt = datetime.datetime.strptime(date_str, date_format) - timezone
         return calendar.timegm(dt.timetuple())
-    except ValueError:
-        pass
 
 
 def date_formats(day_first=True):
@@ -3201,17 +3189,13 @@ def unified_strdate(date_str, day_first=True):
     _, date_str = extract_timezone(date_str)
 
     for expression in date_formats(day_first):
-        try:
+        with compat_contextlib_suppress(ValueError):
             upload_date = datetime.datetime.strptime(date_str, expression).strftime('%Y%m%d')
-        except ValueError:
-            pass
     if upload_date is None:
         timetuple = email.utils.parsedate_tz(date_str)
         if timetuple:
-            try:
+            with compat_contextlib_suppress(ValueError):
                 upload_date = datetime.datetime(*timetuple[:6]).strftime('%Y%m%d')
-            except ValueError:
-                pass
     if upload_date is not None:
         return compat_str(upload_date)
 
@@ -3240,11 +3224,9 @@ def unified_timestamp(date_str, day_first=True):
         date_str = m.group(1)
 
     for expression in date_formats(day_first):
-        try:
+        with compat_contextlib_suppress(ValueError):
             dt = datetime.datetime.strptime(date_str, expression) - timezone + datetime.timedelta(hours=pm_delta)
             return calendar.timegm(dt.timetuple())
-        except ValueError:
-            pass
     timetuple = email.utils.parsedate_tz(date_str)
     if timetuple:
         return calendar.timegm(timetuple) + pm_delta * 3600 - compat_datetime_timedelta_total_seconds(timezone)

From 31a15a7c8dfc618e5090948e6b44d27911383d4b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 10 Mar 2024 19:06:57 +0000
Subject: [PATCH 743/860] [compat] Simplify/fix
 compat_html_parser_HTMLParseError

---
 youtube_dl/compat.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 637f0d82c..75dff58f2 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2421,23 +2421,20 @@ except ImportError:  # Python 2
 compat_urllib_request_urlretrieve = compat_urlretrieve
 
 try:
+    from HTMLParser import (
+        HTMLParser as compat_HTMLParser,
+        HTMLParseError as compat_HTMLParseError)
+except ImportError:  # Python 3
     from html.parser import HTMLParser as compat_HTMLParser
-except ImportError:  # Python 2
-    from HTMLParser import HTMLParser as compat_HTMLParser
-compat_html_parser_HTMLParser = compat_HTMLParser
-
-try:  # Python 2
-    from HTMLParser import HTMLParseError as compat_HTMLParseError
-except ImportError:  # Python <3.4
     try:
         from html.parser import HTMLParseError as compat_HTMLParseError
     except ImportError:  # Python >3.4
-
-        # HTMLParseError has been deprecated in Python 3.3 and removed in
+        # HTMLParseError was deprecated in Python 3.3 and removed in
         # Python 3.5. Introducing dummy exception for Python >3.5 for compatible
         # and uniform cross-version exception handling
         class compat_HTMLParseError(Exception):
             pass
+compat_html_parser_HTMLParser = compat_HTMLParser
 compat_html_parser_HTMLParseError = compat_HTMLParseError
 
 try:

From d8f134a664d7be2c10aba44fc2d54a8f7b0542ff Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 2 Mar 2024 15:17:09 +0000
Subject: [PATCH 744/860] [downloader/external] Fix "Resource Warning" in
 downloader test * add compat_subprocess_Popen context manager * apply context
 manager in FFmpegFD._call_downloader()

---
 youtube_dl/compat.py              | 34 +++++++++++++++++++++++++++---
 youtube_dl/downloader/external.py | 35 ++++++++++++++++++-------------
 2 files changed, 51 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 75dff58f2..53ff2a892 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2438,9 +2438,9 @@ compat_html_parser_HTMLParser = compat_HTMLParser
 compat_html_parser_HTMLParseError = compat_HTMLParseError
 
 try:
-    from subprocess import DEVNULL
-    compat_subprocess_get_DEVNULL = lambda: DEVNULL
-except ImportError:
+    _DEVNULL = subprocess.DEVNULL
+    compat_subprocess_get_DEVNULL = lambda: _DEVNULL
+except AttributeError:
     compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
 
 try:
@@ -2958,6 +2958,33 @@ except ImportError:
             return exc_val is not None and isinstance(exc_val, self._exceptions or tuple())
 
 
+# subprocess.Popen context manager
+# avoids leaking handles if .communicate() is not called
+try:
+    _Popen = subprocess.Popen
+    # check for required context manager attributes
+    _Popen.__enter__ and _Popen.__exit__
+    compat_subprocess_Popen = _Popen
+except AttributeError:
+    # not a context manager - make one
+    from contextlib import contextmanager
+
+    @contextmanager
+    def compat_subprocess_Popen(*args, **kwargs):
+        popen = None
+        try:
+            popen = _Popen(*args, **kwargs)
+            yield popen
+        finally:
+            if popen:
+                for f in (popen.stdin, popen.stdout, popen.stderr):
+                    if f:
+                        # repeated .close() is OK, but just in case
+                        with compat_contextlib_suppress(EnvironmentError):
+                            f.close()
+                popen.wait()
+
+
 # Fix https://github.com/ytdl-org/youtube-dl/issues/4223
 # See http://bugs.python.org/issue9161 for what is broken
 def workaround_optparse_bug9161():
@@ -3314,6 +3341,7 @@ __all__ = [
     'compat_struct_pack',
     'compat_struct_unpack',
     'compat_subprocess_get_DEVNULL',
+    'compat_subprocess_Popen',
     'compat_tokenize_tokenize',
     'compat_urllib_error',
     'compat_urllib_parse',
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index bc228960e..f22fa6013 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -11,6 +11,7 @@ from .common import FileDownloader
 from ..compat import (
     compat_setenv,
     compat_str,
+    compat_subprocess_Popen,
 )
 from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
 from ..utils import (
@@ -483,21 +484,25 @@ class FFmpegFD(ExternalFD):
 
         self._debug_cmd(args)
 
-        proc = subprocess.Popen(args, stdin=subprocess.PIPE, env=env)
-        try:
-            retval = proc.wait()
-        except BaseException as e:
-            # subprocess.run would send the SIGKILL signal to ffmpeg and the
-            # mp4 file couldn't be played, but if we ask ffmpeg to quit it
-            # produces a file that is playable (this is mostly useful for live
-            # streams). Note that Windows is not affected and produces playable
-            # files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
-            if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32':
-                process_communicate_or_kill(proc, b'q')
-            else:
-                proc.kill()
-                proc.wait()
-            raise
+        # From [1], a PIPE opened in Popen() should be closed, unless
+        # .communicate() is called. Avoid leaking any PIPEs by using Popen
+        # as a context manager (newer Python 3.x and compat)
+        # Fixes "Resource Warning" in test/test_downloader_external.py
+        # [1] https://devpress.csdn.net/python/62fde12d7e66823466192e48.html
+        with compat_subprocess_Popen(args, stdin=subprocess.PIPE, env=env) as proc:
+            try:
+                retval = proc.wait()
+            except BaseException as e:
+                # subprocess.run would send the SIGKILL signal to ffmpeg and the
+                # mp4 file couldn't be played, but if we ask ffmpeg to quit it
+                # produces a file that is playable (this is mostly useful for live
+                # streams). Note that Windows is not affected and produces playable
+                # files (see https://github.com/ytdl-org/youtube-dl/issues/8300).
+                if isinstance(e, KeyboardInterrupt) and sys.platform != 'win32':
+                    process_communicate_or_kill(proc, b'q')
+                else:
+                    proc.kill()
+                raise
         return retval
 
 

From 21792b88b791b16e3ab0a0fb2e26e5bb8a4e2ff3 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 3 Mar 2024 12:38:00 +0000
Subject: [PATCH 745/860] [external/FFmpeg] Fix and improve --ffmpeg-location
 handling * pass YoutubeDL (FileDownloader) to FFmpegPostProcessor constructor
 * consolidate path search in FFmpegPostProcessor * make availability of
 FFmpegFD depend on existence of FFmpegPostProcessor * detect ffmpeg
 executable on instantiation of FFmpegFD * resolves #32735

---
 test/test_downloader_external.py   | 16 ++++++++++++-
 youtube_dl/downloader/external.py  | 17 +++++++++-----
 youtube_dl/postprocessor/ffmpeg.py | 36 +++++++++---------------------
 3 files changed, 38 insertions(+), 31 deletions(-)

diff --git a/test/test_downloader_external.py b/test/test_downloader_external.py
index 029f9b05f..4491bd9de 100644
--- a/test/test_downloader_external.py
+++ b/test/test_downloader_external.py
@@ -18,6 +18,7 @@ from test.helper import (
 )
 from youtube_dl import YoutubeDL
 from youtube_dl.compat import (
+    compat_contextlib_suppress,
     compat_http_cookiejar_Cookie,
     compat_http_server,
     compat_kwargs,
@@ -35,6 +36,9 @@ from youtube_dl.downloader.external import (
     HttpieFD,
     WgetFD,
 )
+from youtube_dl.postprocessor import (
+    FFmpegPostProcessor,
+)
 import threading
 
 TEST_SIZE = 10 * 1024
@@ -227,7 +231,17 @@ class TestAria2cFD(unittest.TestCase):
             self.assertIn('--load-cookies=%s' % downloader._cookies_tempfile, cmd)
 
 
-@ifExternalFDAvailable(FFmpegFD)
+# Handle delegated availability
+def ifFFmpegFDAvailable(externalFD):
+    # raise SkipTest, or set False!
+    avail = ifExternalFDAvailable(externalFD) and False
+    with compat_contextlib_suppress(Exception):
+        avail = FFmpegPostProcessor(downloader=None).available
+    return unittest.skipUnless(
+        avail, externalFD.get_basename() + ' not found')
+
+
+@ifFFmpegFDAvailable(FFmpegFD)
 class TestFFmpegFD(unittest.TestCase):
     _args = []
 
diff --git a/youtube_dl/downloader/external.py b/youtube_dl/downloader/external.py
index f22fa6013..4fbc0f520 100644
--- a/youtube_dl/downloader/external.py
+++ b/youtube_dl/downloader/external.py
@@ -13,7 +13,12 @@ from ..compat import (
     compat_str,
     compat_subprocess_Popen,
 )
-from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
+
+try:
+    from ..postprocessor.ffmpeg import FFmpegPostProcessor, EXT_TO_OUT_FORMATS
+except ImportError:
+    FFmpegPostProcessor = None
+
 from ..utils import (
     cli_option,
     cli_valueless_option,
@@ -362,13 +367,14 @@ class FFmpegFD(ExternalFD):
 
     @classmethod
     def available(cls):
-        return FFmpegPostProcessor().available
+        # actual availability can only be confirmed for an instance
+        return bool(FFmpegPostProcessor)
 
     def _call_downloader(self, tmpfilename, info_dict):
-        url = info_dict['url']
-        ffpp = FFmpegPostProcessor(downloader=self)
+        # `downloader` means the parent `YoutubeDL`
+        ffpp = FFmpegPostProcessor(downloader=self.ydl)
         if not ffpp.available:
-            self.report_error('m3u8 download detected but ffmpeg or avconv could not be found. Please install one.')
+            self.report_error('ffmpeg required for download but no ffmpeg (nor avconv) executable could be found. Please install one.')
             return False
         ffpp.check_version()
 
@@ -397,6 +403,7 @@ class FFmpegFD(ExternalFD):
         # if end_time:
         #     args += ['-t', compat_str(end_time - start_time)]
 
+        url = info_dict['url']
         cookies = self.ydl.cookiejar.get_cookies_for_url(url)
         if cookies:
             args.extend(['-cookies', ''.join(
diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index 801160e6c..e5ffdf378 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -96,6 +96,7 @@ class FFmpegPostProcessor(PostProcessor):
 
         self._paths = None
         self._versions = None
+        location = None
         if self._downloader:
             prefer_ffmpeg = self._downloader.params.get('prefer_ffmpeg', True)
             location = self._downloader.params.get('ffmpeg_location')
@@ -118,32 +119,17 @@ class FFmpegPostProcessor(PostProcessor):
                     location = os.path.dirname(os.path.abspath(location))
                     if basename in ('ffmpeg', 'ffprobe'):
                         prefer_ffmpeg = True
+        self._paths = dict(
+            (p, p if location is None else os.path.join(location, p))
+            for p in programs)
+        self._versions = dict(
+            x for x in (
+                (p, get_ffmpeg_version(self._paths[p])) for p in programs)
+            if x[1] is not None)
 
-                self._paths = dict(
-                    (p, os.path.join(location, p)) for p in programs)
-                self._versions = dict(
-                    (p, get_ffmpeg_version(self._paths[p])) for p in programs)
-        if self._versions is None:
-            self._versions = dict(
-                (p, get_ffmpeg_version(p)) for p in programs)
-            self._paths = dict((p, p) for p in programs)
-
-        if prefer_ffmpeg is False:
-            prefs = ('avconv', 'ffmpeg')
-        else:
-            prefs = ('ffmpeg', 'avconv')
-        for p in prefs:
-            if self._versions[p]:
-                self.basename = p
-                break
-
-        if prefer_ffmpeg is False:
-            prefs = ('avprobe', 'ffprobe')
-        else:
-            prefs = ('ffprobe', 'avprobe')
-        for p in prefs:
-            if self._versions[p]:
-                self.probe_basename = p
+        for p in ('ffmpeg', 'avconv')[::-1 if prefer_ffmpeg is False else 1]:
+            if self._versions.get(p):
+                self.basename = self.probe_basename = p
                 break
 
     @property

From 4ea59c6107d63bbabe1de74569ceb984b1646426 Mon Sep 17 00:00:00 2001
From: Ori Avtalion <ori@avtalion.name>
Date: Fri, 5 Apr 2024 17:25:29 +0300
Subject: [PATCH 746/860] [utils] Fix crash in _report_ignoring_subs from
 c58b655  (#32762)

Align `utils.bug_reports_message()` with yt-dlp https://github.com/yt-dlp/yt-dlp/commit/5873d4ccdd, thanks fstirlitz

---------

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/utils.py | 21 +++++++++++++++------
 1 file changed, 15 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 083446342..e1b05b307 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2371,15 +2371,24 @@ def make_HTTPS_handler(params, **kwargs):
         return YoutubeDLHTTPSHandler(params, context=context, **kwargs)
 
 
-def bug_reports_message():
+def bug_reports_message(before=';'):
     if ytdl_is_updateable():
         update_cmd = 'type  youtube-dl -U  to update'
     else:
-        update_cmd = 'see  https://yt-dl.org/update  on how to update'
-    msg = '; please report this issue on https://yt-dl.org/bug .'
-    msg += ' Make sure you are using the latest version; %s.' % update_cmd
-    msg += ' Be sure to call youtube-dl with the --verbose flag and include its complete output.'
-    return msg
+        update_cmd = 'see  https://github.com/ytdl-org/youtube-dl/#user-content-installation  on how to update'
+
+    msg = (
+        'please report this issue on https://github.com/ytdl-org/youtube-dl/issues ,'
+        ' using the appropriate issue template.'
+        ' Make sure you are using the latest version; %s.'
+        ' Be sure to call youtube-dl with the --verbose option and include the complete output.'
+    ) % update_cmd
+
+    before = (before or '').rstrip()
+    if not before or before.endswith(('.', '!', '?')):
+        msg = msg[0].title() + msg[1:]
+
+    return (before + ' ' if before else '') + msg
 
 
 class YoutubeDLError(Exception):

From e0727e4ab61b6e45f7792546b8b5ff52a0ea22b5 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 7 Apr 2024 15:26:12 +0100
Subject: [PATCH 747/860] [postprocessor/ffmpeg] Fix finding ffprobe (bug in
 21792b8) Fixes
 https://github.com/ytdl-org/youtube-dl/commit/21792b88b791b16e3ab0a0fb2e26e5bb8a4e2ff3#commitcomment-140705274,
 thx: vonProteus

---
 youtube_dl/postprocessor/ffmpeg.py | 16 +++++++++++-----
 1 file changed, 11 insertions(+), 5 deletions(-)

diff --git a/youtube_dl/postprocessor/ffmpeg.py b/youtube_dl/postprocessor/ffmpeg.py
index e5ffdf378..214825aa9 100644
--- a/youtube_dl/postprocessor/ffmpeg.py
+++ b/youtube_dl/postprocessor/ffmpeg.py
@@ -74,8 +74,11 @@ class FFmpegPostProcessor(PostProcessor):
         return FFmpegPostProcessor(downloader)._versions
 
     def _determine_executables(self):
-        programs = ['avprobe', 'avconv', 'ffmpeg', 'ffprobe']
+        # ordered to match prefer_ffmpeg!
+        convs = ['ffmpeg', 'avconv']
+        probes = ['ffprobe', 'avprobe']
         prefer_ffmpeg = True
+        programs = convs + probes
 
         def get_ffmpeg_version(path):
             ver = get_exe_version(path, args=['-version'])
@@ -127,10 +130,13 @@ class FFmpegPostProcessor(PostProcessor):
                 (p, get_ffmpeg_version(self._paths[p])) for p in programs)
             if x[1] is not None)
 
-        for p in ('ffmpeg', 'avconv')[::-1 if prefer_ffmpeg is False else 1]:
-            if self._versions.get(p):
-                self.basename = self.probe_basename = p
-                break
+        basenames = [None, None]
+        for i, progs in enumerate((convs, probes)):
+            for p in progs[::-1 if prefer_ffmpeg is False else 1]:
+                if self._versions.get(p):
+                    basenames[i] = p
+                    break
+        self.basename, self.probe_basename = basenames
 
     @property
     def available(self):

From eb3866543810c7be96f40ec94e0c00df19b93861 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 1 Mar 2024 14:48:30 +0000
Subject: [PATCH 748/860] [YouPorn] Incorporate yt-dlp PR 8827 * from
 https://github.com/yt-dlp/yt-dlp/pull/8827 * extract from webpage instead of
 broken API URL * thx The-MAGI

---
 youtube_dl/extractor/youporn.py | 118 ++++++++++++++++++++++----------
 1 file changed, 80 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 31e8abb72..4c455b5b8 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -8,6 +8,9 @@ from ..utils import (
     extract_attributes,
     int_or_none,
     str_to_int,
+    merge_dicts,
+    T,
+    traverse_obj,
     unified_strdate,
     url_or_none,
 )
@@ -15,6 +18,7 @@ from ..utils import (
 
 class YouPornIE(InfoExtractor):
     _VALID_URL = r'https?://(?:www\.)?youporn\.com/(?:watch|embed)/(?P<id>\d+)(?:/(?P<display_id>[^/?#&]+))?'
+    _EMBED_REGEX = [r'<iframe[^>]+\bsrc=["\'](?P<url>(?:https?:)?//(?:www\.)?youporn\.com/embed/\d+)']
     _TESTS = [{
         'url': 'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
         'md5': '3744d24c50438cf5b6f6d59feb5055c2',
@@ -66,57 +70,91 @@ class YouPornIE(InfoExtractor):
     }, {
         'url': 'https://www.youporn.com/watch/13922959/femdom-principal/',
         'only_matching': True,
+    }, {
+        'url': 'https://www.youporn.com/watch/16290308/tinderspecial-trailer1/',
+        'info_dict': {
+            'id': '46949121',
+            'age_limit': 18,
+            'categories': [],
+            'description': None,  # SEO spam using title removed
+            'display_id': 'tinderspecial-trailer1',
+            'duration': 298.0,
+            'ext': 'mp4',
+            'upload_date': '20201123',
+            'uploader': 'Ersties',
+            'tags': [],
+            'thumbnail': 'https://fi1.ypncdn.com/m=eaSaaTbWx/202011/23/16290308/original/3.jpg',
+            'timestamp': 1606147564,
+            'title': 'Tinder In Real Life',
+            'view_count': int,
+        }
     }]
 
-    @staticmethod
-    def _extract_urls(webpage):
-        return re.findall(
-            r'<iframe[^>]+\bsrc=["\']((?:https?:)?//(?:www\.)?youporn\.com/embed/\d+)',
-            webpage)
+    @classmethod
+    def _extract_urls(cls, webpage):
+        def yield_urls():
+            for p in cls._EMBED_REGEX:
+                for m in re.finditer(p, webpage):
+                    yield m.group('url')
+
+        return list(yield_urls())
 
     def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        video_id = mobj.group('id')
-        display_id = mobj.group('display_id') or video_id
+        display_id = self._match_valid_url(url).group('id', 'display_id')
+        url = 'http://www.youporn.com/watch/%s' % (display_id[0],)
+        display_id = display_id[1] or display_id[0]
+        webpage = self._download_webpage(
+            url, display_id, headers={'Cookie': 'age_verified=1'})
+        video_id = display_id[0]
 
-        definitions = self._download_json(
-            'https://www.youporn.com/api/video/media_definitions/%s/' % video_id,
-            display_id)
+        playervars = self._search_json(
+            r'\bplayervars\s*:', webpage, 'playervars', display_id)
+
+        def get_fmt(x):
+            v_url = url_or_none(x.get('videoUrl'))
+            if v_url:
+                x['videoUrl'] = v_url
+                return (x['format'], x)
+
+        defs_by_format = dict(traverse_obj(playervars, (
+            'mediaDefinitions', lambda _, v: v.get('format'), T(get_fmt))))
+
+        def get_format_data(f):
+            if f not in defs_by_format:
+                return []
+            return self._download_json(
+                defs_by_format[f]['videoUrl'], display_id, '{0}-formats'.format(f))
 
         formats = []
-        for definition in definitions:
-            if not isinstance(definition, dict):
-                continue
-            video_url = url_or_none(definition.get('videoUrl'))
-            if not video_url:
-                continue
-            f = {
-                'url': video_url,
-                'filesize': int_or_none(definition.get('videoSize')),
-            }
-            height = int_or_none(definition.get('quality'))
+        # Try to extract only the actual master m3u8 first, avoiding the duplicate single resolution "master" m3u8s
+        for hls_url in traverse_obj(
+                get_format_data('hls'),
+                (lambda _, v: not isinstance(v['defaultQuality'], bool), 'videoUrl'),
+                (Ellipsis, 'videoUrl')):
+            formats.extend(self._extract_m3u8_formats(
+                hls_url, video_id, 'mp4', fatal=False, m3u8_id='hls',
+                entry_protocol='m3u8_native'))
+
+        for f in traverse_obj(get_format_data('mp4'), (
+                lambda _, v: v.get('videoUrl'), {
+                    'url': ('videoUrl', T(url_or_none)),
+                    'filesize': ('videoSize', T(int_or_none)),
+                    'height': ('quality', T(int_or_none)),
+                }, T(lambda x: x.get('videoUrl') and x))):
             # Video URL's path looks like this:
             #  /201012/17/505835/720p_1500k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
             #  /201012/17/505835/vl_240p_240k_505835/YouPorn%20-%20Sex%20Ed%20Is%20It%20Safe%20To%20Masturbate%20Daily.mp4
             #  /videos/201703/11/109285532/1080P_4000K_109285532.mp4
             # We will benefit from it by extracting some metadata
-            mobj = re.search(r'(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+', video_url)
+            mobj = re.search(r'(?P<height>\d{3,4})[pP]_(?P<bitrate>\d+)[kK]_\d+', f['videoUrl'])
             if mobj:
-                if not height:
-                    height = int(mobj.group('height'))
-                bitrate = int(mobj.group('bitrate'))
-                f.update({
-                    'format_id': '%dp-%dk' % (height, bitrate),
-                    'tbr': bitrate,
-                })
-            f['height'] = height
+                if not f.get('height'):
+                    f['height'] = int(mobj.group('height'))
+                f['tbr'] = int(mobj.group('bitrate'))
+                f['format_id'] = '%dp-%dk' % (f['height'], f['tbr'])
             formats.append(f)
         self._sort_formats(formats)
 
-        webpage = self._download_webpage(
-            'http://www.youporn.com/watch/%s' % video_id, display_id,
-            headers={'Cookie': 'age_verified=1'})
-
         title = self._html_search_regex(
             r'(?s)<div[^>]+class=["\']watchVideoTitle[^>]+>(.+?)</div>',
             webpage, 'title', default=None) or self._og_search_title(
@@ -168,9 +206,12 @@ class YouPornIE(InfoExtractor):
             r'(?s)Tags:.*?</div>\s*<div[^>]+class=["\']tagBoxContent["\'][^>]*>(.+?)</div>',
             'tags')
 
-        return {
+        data = self._search_json_ld(webpage, video_id, expected_type='VideoObject', fatal=False) or {}
+        data.pop('url', None)
+
+        result = merge_dicts(data, {
             'id': video_id,
-            'display_id': display_id,
+            'display_id': display_id if display_id != video_id else None,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
@@ -183,4 +224,5 @@ class YouPornIE(InfoExtractor):
             'tags': tags,
             'age_limit': age_limit,
             'formats': formats,
-        }
+        })
+        return result

From c2766cb80ee9add4d2d2d94b1878e7884c63ceaf Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 11 Apr 2024 12:47:05 +0100
Subject: [PATCH 749/860] [test/test_download] Support
 'playlist_maxcount:count' expected value * parallel to `playlist_mincount' *
 specify both for a range of playlist lengths * if max < min the test will
 always fail!

---
 test/test_download.py | 14 +++++++++++++-
 1 file changed, 13 insertions(+), 1 deletion(-)

diff --git a/test/test_download.py b/test/test_download.py
index e0bc8cb95..df8b370cf 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -10,6 +10,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from test.helper import (
     assertGreaterEqual,
+    assertLessEqual,
     expect_warnings,
     get_params,
     gettestcases,
@@ -122,7 +123,10 @@ def generator(test_case, tname):
         params['outtmpl'] = tname + '_' + params['outtmpl']
         if is_playlist and 'playlist' not in test_case:
             params.setdefault('extract_flat', 'in_playlist')
-            params.setdefault('playlistend', test_case.get('playlist_mincount'))
+            params.setdefault('playlistend',
+                              test_case['playlist_maxcount'] + 1
+                              if test_case.get('playlist_maxcount')
+                              else test_case.get('playlist_mincount'))
             params.setdefault('skip_download', True)
 
         ydl = YoutubeDL(params, auto_init=False)
@@ -190,6 +194,14 @@ def generator(test_case, tname):
                     'Expected at least %d in playlist %s, but got only %d' % (
                         test_case['playlist_mincount'], test_case['url'],
                         len(res_dict['entries'])))
+            if 'playlist_maxcount' in test_case:
+                assertLessEqual(
+                    self,
+                    len(res_dict['entries']),
+                    test_case['playlist_maxcount'],
+                    'Expected at most %d in playlist %s, but got %d' % (
+                        test_case['playlist_maxcount'], test_case['url'],
+                        len(res_dict['entries'])))
             if 'playlist_count' in test_case:
                 self.assertEqual(
                     len(res_dict['entries']),

From 0b2ce3685e02ea1a3ccee1026572e081b8f6ac83 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 1 Mar 2024 15:07:30 +0000
Subject: [PATCH 750/860] [YouPorn] Improve extraction * detect unwatchable
 videos * improve duration extraction * fix count extraction and support large
 values * detect and remove SEO spam boilerplate description

---
 youtube_dl/extractor/youporn.py | 64 +++++++++++++++++++++++----------
 1 file changed, 46 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 4c455b5b8..8f2175dde 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -5,10 +5,13 @@ import re
 
 from .common import InfoExtractor
 from ..utils import (
+    clean_html,
     extract_attributes,
+    ExtractorError,
+    get_element_by_id,
     int_or_none,
-    str_to_int,
     merge_dicts,
+    parse_count,
     T,
     traverse_obj,
     unified_strdate,
@@ -17,7 +20,13 @@ from ..utils import (
 
 
 class YouPornIE(InfoExtractor):
-    _VALID_URL = r'https?://(?:www\.)?youporn\.com/(?:watch|embed)/(?P<id>\d+)(?:/(?P<display_id>[^/?#&]+))?'
+    _VALID_URL = (
+        r'youporn:(?P<id>\d+)',
+        r'''(?x)
+            https?://(?:www\.)?youporn\.com/(?:watch|embed)/(?P<id>\d+)
+            (?:/(?:(?P<display_id>[^/?#&]+)/?)?)?(?:[#?]|$)
+    '''
+    )
     _EMBED_REGEX = [r'<iframe[^>]+\bsrc=["\'](?P<url>(?:https?:)?//(?:www\.)?youporn\.com/embed/\d+)']
     _TESTS = [{
         'url': 'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/',
@@ -38,7 +47,7 @@ class YouPornIE(InfoExtractor):
             'tags': list,
             'age_limit': 18,
         },
-        'skip': 'This video has been disabled',
+        'skip': 'This video has been deactivated',
     }, {
         # Unknown uploader
         'url': 'http://www.youporn.com/watch/561726/big-tits-awesome-brunette-on-amazing-webcam-show/?from=related3&al=2&from_id=561726&pos=4',
@@ -73,7 +82,7 @@ class YouPornIE(InfoExtractor):
     }, {
         'url': 'https://www.youporn.com/watch/16290308/tinderspecial-trailer1/',
         'info_dict': {
-            'id': '46949121',
+            'id': '16290308',
             'age_limit': 18,
             'categories': [],
             'description': None,  # SEO spam using title removed
@@ -100,15 +109,28 @@ class YouPornIE(InfoExtractor):
         return list(yield_urls())
 
     def _real_extract(self, url):
-        display_id = self._match_valid_url(url).group('id', 'display_id')
-        url = 'http://www.youporn.com/watch/%s' % (display_id[0],)
-        display_id = display_id[1] or display_id[0]
+        # A different video ID (data-video-id) is hidden in the page but
+        # never seems to be used
+        video_id, display_id = self._match_valid_url(url).group('id', 'display_id')
+        url = 'http://www.youporn.com/watch/%s' % (video_id,)
         webpage = self._download_webpage(
-            url, display_id, headers={'Cookie': 'age_verified=1'})
-        video_id = display_id[0]
+            url, video_id, headers={'Cookie': 'age_verified=1'})
+
+        watchable = self._search_regex(
+            r'''(<div\s[^>]*\bid\s*=\s*('|")?watch-container(?(2)\2|(?!-)\b)[^>]*>)''',
+            webpage, 'watchability', default=None)
+        if not watchable:
+            msg = re.split(r'\s{4}', clean_html(get_element_by_id(
+                'mainContent', webpage)) or '')[0]
+            raise ExtractorError(
+                ('%s says: %s' % (self.IE_NAME, msg))
+                if msg else 'Video unavailable: no reason found',
+                expected=True)
+        # internal ID ?
+        # video_id = extract_attributes(watchable).get('data-video-id')
 
         playervars = self._search_json(
-            r'\bplayervars\s*:', webpage, 'playervars', display_id)
+            r'\bplayervars\s*:', webpage, 'playervars', video_id)
 
         def get_fmt(x):
             v_url = url_or_none(x.get('videoUrl'))
@@ -123,7 +145,7 @@ class YouPornIE(InfoExtractor):
             if f not in defs_by_format:
                 return []
             return self._download_json(
-                defs_by_format[f]['videoUrl'], display_id, '{0}-formats'.format(f))
+                defs_by_format[f]['videoUrl'], video_id, '{0}-formats'.format(f))
 
         formats = []
         # Try to extract only the actual master m3u8 first, avoiding the duplicate single resolution "master" m3u8s
@@ -169,8 +191,10 @@ class YouPornIE(InfoExtractor):
         thumbnail = self._search_regex(
             r'(?:imageurl\s*=|poster\s*:)\s*(["\'])(?P<thumbnail>.+?)\1',
             webpage, 'thumbnail', fatal=False, group='thumbnail')
-        duration = int_or_none(self._html_search_meta(
-            'video:duration', webpage, 'duration', fatal=False))
+        duration = traverse_obj(playervars, ('duration', T(int_or_none)))
+        if duration is None:
+            duration = int_or_none(self._html_search_meta(
+                'video:duration', webpage, 'duration', fatal=False))
 
         uploader = self._html_search_regex(
             r'(?s)<div[^>]+class=["\']submitByLink["\'][^>]*>(.+?)</div>',
@@ -186,11 +210,11 @@ class YouPornIE(InfoExtractor):
 
         view_count = None
         views = self._search_regex(
-            r'(<div[^>]+\bclass=["\']js_videoInfoViews["\']>)', webpage,
-            'views', default=None)
+            r'(<div\s[^>]*\bdata-value\s*=[^>]+>)\s*<label>Views:</label>',
+            webpage, 'views', default=None)
         if views:
-            view_count = str_to_int(extract_attributes(views).get('data-value'))
-        comment_count = str_to_int(self._search_regex(
+            view_count = parse_count(extract_attributes(views).get('data-value'))
+        comment_count = parse_count(self._search_regex(
             r'>All [Cc]omments? \(([\d,.]+)\)',
             webpage, 'comment count', default=None))
 
@@ -211,7 +235,7 @@ class YouPornIE(InfoExtractor):
 
         result = merge_dicts(data, {
             'id': video_id,
-            'display_id': display_id if display_id != video_id else None,
+            'display_id': display_id,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,
@@ -225,4 +249,8 @@ class YouPornIE(InfoExtractor):
             'age_limit': age_limit,
             'formats': formats,
         })
+        # Remove promotional non-description
+        if result.get('description', '').startswith(
+                'Watch %s online' % (result['title'],)):
+            del result['description']
         return result

From 668332b9733023ca2e927eeb2208725022248af8 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 11 Apr 2024 13:06:25 +0100
Subject: [PATCH 751/860] [YouPorn] Add playlist extractors * YouPornCategoryIE
 * YouPornChannelIE * YouPornCollectionIE * YouPornStarIE * YouPornTagIE *
 YouPornVideosIE,

---
 youtube_dl/extractor/extractors.py |  10 +-
 youtube_dl/extractor/youporn.py    | 438 +++++++++++++++++++++++++++++
 2 files changed, 447 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index a56a7c52f..03d035a27 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -1653,7 +1653,15 @@ from .younow import (
     YouNowChannelIE,
     YouNowMomentIE,
 )
-from .youporn import YouPornIE
+from .youporn import (
+    YouPornIE,
+    YouPornCategoryIE,
+    YouPornChannelIE,
+    YouPornCollectionIE,
+    YouPornStarIE,
+    YouPornTagIE,
+    YouPornVideosIE,
+)
 from .yourporn import YourPornIE
 from .yourupload import YourUploadIE
 from .youtube import (
diff --git a/youtube_dl/extractor/youporn.py b/youtube_dl/extractor/youporn.py
index 8f2175dde..ec6125a79 100644
--- a/youtube_dl/extractor/youporn.py
+++ b/youtube_dl/extractor/youporn.py
@@ -1,21 +1,26 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import itertools
 import re
+from time import sleep
 
 from .common import InfoExtractor
 from ..utils import (
     clean_html,
     extract_attributes,
     ExtractorError,
+    get_element_by_class,
     get_element_by_id,
     int_or_none,
     merge_dicts,
     parse_count,
+    parse_qs,
     T,
     traverse_obj,
     unified_strdate,
     url_or_none,
+    urljoin,
 )
 
 
@@ -254,3 +259,436 @@ class YouPornIE(InfoExtractor):
                 'Watch %s online' % (result['title'],)):
             del result['description']
         return result
+
+
+class YouPornListBase(InfoExtractor):
+    # pattern in '.title-text' element of page section containing videos
+    _PLAYLIST_TITLEBAR_RE = r'\s+[Vv]ideos\s*$'
+    _PAGE_RETRY_COUNT = 0  # ie, no retry
+    _PAGE_RETRY_DELAY = 2  # seconds
+
+    def _get_next_url(self, url, pl_id, html):
+        return urljoin(url, self._search_regex(
+            r'''<a\s[^>]*?\bhref\s*=\s*("|')(?P<url>(?:(?!\1)[^>])+)\1''',
+            get_element_by_id('next', html) or '', 'next page',
+            group='url', default=None))
+
+    @classmethod
+    def _get_title_from_slug(cls, title_slug):
+        return re.sub(r'[_-]', ' ', title_slug)
+
+    def _entries(self, url, pl_id, html=None, page_num=None):
+
+        # separates page sections
+        PLAYLIST_SECTION_RE = (
+            r'''<div\s[^>]*\bclass\s*=\s*('|")(?:[\w$-]+\s+|\s)*?title-bar(?:\s+[\w$-]+|\s)*\1[^>]*>'''
+        )
+        # contains video link
+        VIDEO_URL_RE = r'''(?x)
+            <div\s[^>]*\bdata-video-id\s*=\s*('|")\d+\1[^>]*>\s*
+            (?:<div\b[\s\S]+?</div>\s*)*
+            <a\s[^>]*\bhref\s*=\s*('|")(?P<url>(?:(?!\2)[^>])+)\2
+        '''
+
+        def yield_pages(url, html=html, page_num=page_num):
+            fatal = not html
+            for pnum in itertools.count(start=page_num or 1):
+                if not html:
+                    html = self._download_webpage(
+                        url, pl_id, note='Downloading page %d' % pnum,
+                        fatal=fatal)
+                if not html:
+                    break
+                fatal = False
+                yield (url, html, pnum)
+                # explicit page: extract just that page
+                if page_num is not None:
+                    break
+                next_url = self._get_next_url(url, pl_id, html)
+                if not next_url or next_url == url:
+                    break
+                url, html = next_url, None
+
+        def retry_page(msg, tries_left, page_data):
+            if tries_left <= 0:
+                return
+            self.report_warning(msg, pl_id)
+            sleep(self._PAGE_RETRY_DELAY)
+            return next(
+                yield_pages(page_data[0], page_num=page_data[2]), None)
+
+        def yield_entries(html):
+            for frag in re.split(PLAYLIST_SECTION_RE, html):
+                if not frag:
+                    continue
+                t_text = get_element_by_class('title-text', frag or '')
+                if not (t_text and re.search(self._PLAYLIST_TITLEBAR_RE, t_text)):
+                    continue
+                for m in re.finditer(VIDEO_URL_RE, frag):
+                    video_url = urljoin(url, m.group('url'))
+                    if video_url:
+                        yield self.url_result(video_url)
+
+        last_first_url = None
+        for page_data in yield_pages(url, html=html, page_num=page_num):
+            # page_data: url, html, page_num
+            first_url = None
+            tries_left = self._PAGE_RETRY_COUNT + 1
+            while tries_left > 0:
+                tries_left -= 1
+                for from_ in yield_entries(page_data[1]):
+                    # may get the same page twice instead of empty page
+                    # or (site bug) intead of actual next page
+                    if not first_url:
+                        first_url = from_['url']
+                        if first_url == last_first_url:
+                            # sometimes (/porntags/) the site serves the previous page
+                            # instead but may provide the correct page after a delay
+                            page_data = retry_page(
+                                'Retrying duplicate page...', tries_left, page_data)
+                            if page_data:
+                                first_url = None
+                                break
+                            continue
+                    yield from_
+                else:
+                    if not first_url and 'no-result-paragarph1' in page_data[1]:
+                        page_data = retry_page(
+                            'Retrying empty page...', tries_left, page_data)
+                        if page_data:
+                            continue
+                    else:
+                        # success/failure
+                        break
+            # may get an infinite (?) sequence of empty pages
+            if not first_url:
+                break
+            last_first_url = first_url
+
+    def _real_extract(self, url, html=None):
+        # exceptionally, id may be None
+        m_dict = self._match_valid_url(url).groupdict()
+        pl_id, page_type, sort = (m_dict.get(k) for k in ('id', 'type', 'sort'))
+
+        qs = parse_qs(url)
+        for q, v in qs.items():
+            if v:
+                qs[q] = v[-1]
+            else:
+                del qs[q]
+
+        base_id = pl_id or 'YouPorn'
+        title = self._get_title_from_slug(base_id)
+        if page_type:
+            title = '%s %s' % (page_type.capitalize(), title)
+        base_id = [base_id.lower()]
+        if sort is None:
+            title += ' videos'
+        else:
+            title = '%s videos by %s' % (title, re.sub(r'[_-]', ' ', sort))
+            base_id.append(sort)
+        if qs:
+            ps = ['%s=%s' % item for item in sorted(qs.items())]
+            title += ' (%s)' % ','.join(ps)
+            base_id.extend(ps)
+        pl_id = '/'.join(base_id)
+
+        return self.playlist_result(
+            self._entries(url, pl_id, html=html,
+                          page_num=int_or_none(qs.get('page'))),
+            playlist_id=pl_id, playlist_title=title)
+
+
+class YouPornCategoryIE(YouPornListBase):
+    IE_DESC = 'YouPorn category, with sorting, filtering and pagination'
+    _VALID_URL = r'''(?x)
+        https?://(?:www\.)?youporn\.com/
+        (?P<type>category)/(?P<id>[^/?#&]+)
+        (?:/(?P<sort>popular|views|rating|time|duration))?/?(?:[#?]|$)
+    '''
+    _TESTS = [{
+        'note': 'Full list with pagination',
+        'url': 'https://www.youporn.com/category/lingerie/popular/',
+        'info_dict': {
+            'id': 'lingerie/popular',
+            'title': 'Category lingerie videos by popular',
+        },
+        'playlist_mincount': 39,
+    }, {
+        'note': 'Filtered paginated list with single page result',
+        'url': 'https://www.youporn.com/category/lingerie/duration/?min_minutes=10',
+        'info_dict': {
+            'id': 'lingerie/duration/min_minutes=10',
+            'title': 'Category lingerie videos by duration (min_minutes=10)',
+        },
+        'playlist_maxcount': 30,
+    }, {
+        'note': 'Single page of full list',
+        'url': 'https://www.youporn.com/category/lingerie/popular?page=1',
+        'info_dict': {
+            'id': 'lingerie/popular/page=1',
+            'title': 'Category lingerie videos by popular (page=1)',
+        },
+        'playlist_count': 30,
+    }]
+
+
+class YouPornChannelIE(YouPornListBase):
+    IE_DESC = 'YouPorn channel, with sorting and pagination'
+    _VALID_URL = r'''(?x)
+        https?://(?:www\.)?youporn\.com/
+        (?P<type>channel)/(?P<id>[^/?#&]+)
+        (?:/(?P<sort>rating|views|duration))?/?(?:[#?]|$)
+    '''
+    _TESTS = [{
+        'note': 'Full list with pagination',
+        'url': 'https://www.youporn.com/channel/x-feeds/',
+        'info_dict': {
+            'id': 'x-feeds',
+            'title': 'Channel X-Feeds videos',
+        },
+        'playlist_mincount': 37,
+    }, {
+        'note': 'Single page of full list (no filters here)',
+        'url': 'https://www.youporn.com/channel/x-feeds/duration?page=1',
+        'info_dict': {
+            'id': 'x-feeds/duration/page=1',
+            'title': 'Channel X-Feeds videos by duration (page=1)',
+        },
+        'playlist_count': 24,
+    }]
+
+    @staticmethod
+    def _get_title_from_slug(title_slug):
+        return re.sub(r'_', ' ', title_slug).title()
+
+
+class YouPornCollectionIE(YouPornListBase):
+    IE_DESC = 'YouPorn collection (user playlist), with sorting and pagination'
+    _VALID_URL = r'''(?x)
+        https?://(?:www\.)?youporn\.com/
+        (?P<type>collection)s/videos/(?P<id>\d+)
+        (?:/(?P<sort>rating|views|time|duration))?/?(?:[#?]|$)
+    '''
+    _PLAYLIST_TITLEBAR_RE = r'^\s*Videos\s+in\s'
+    _TESTS = [{
+        'note': 'Full list with pagination',
+        'url': 'https://www.youporn.com/collections/videos/33044251/',
+        'info_dict': {
+            'id': '33044251',
+            'title': 'Collection Sexy Lips videos',
+            'uploader': 'ph-littlewillyb',
+        },
+        'playlist_mincount': 50,
+    }, {
+        'note': 'Single page of full list (no filters here)',
+        'url': 'https://www.youporn.com/collections/videos/33044251/time?page=1',
+        'info_dict': {
+            'id': '33044251/time/page=1',
+            'title': 'Collection Sexy Lips videos by time (page=1)',
+            'uploader': 'ph-littlewillyb',
+        },
+        'playlist_count': 20,
+    }]
+
+    def _real_extract(self, url):
+        pl_id = self._match_id(url)
+        html = self._download_webpage(url, pl_id)
+        playlist = super(YouPornCollectionIE, self)._real_extract(url, html=html)
+        infos = re.sub(r'\s+', ' ', clean_html(get_element_by_class(
+            'collection-infos', html)) or '')
+        title, uploader = self._search_regex(
+            r'^\s*Collection: (?P<title>.+?) \d+ VIDEOS \d+ VIEWS \d+ days LAST UPDATED From: (?P<uploader>[\w_-]+)',
+            infos, 'title/uploader', group=('title', 'uploader'), default=(None, None))
+
+        return merge_dicts({
+            'title': playlist['title'].replace(playlist['id'].split('/')[0], title),
+            'uploader': uploader,
+        }, playlist) if title else playlist
+
+
+class YouPornTagIE(YouPornListBase):
+    IE_DESC = 'YouPorn tag (porntags), with sorting, filtering and pagination'
+    _VALID_URL = r'''(?x)
+        https?://(?:www\.)?youporn\.com/
+        porn(?P<type>tag)s/(?P<id>[^/?#&]+)
+        (?:/(?P<sort>views|rating|time|duration))?/?(?:[#?]|$)
+    '''
+    _PLAYLIST_TITLEBAR_RE = r'^\s*Videos\s+tagged\s'
+    _PAGE_RETRY_COUNT = 1
+    _TESTS = [{
+        'note': 'Full list with pagination',
+        'url': 'https://www.youporn.com/porntags/austrian',
+        'info_dict': {
+            'id': 'austrian',
+            'title': 'Tag austrian videos',
+        },
+        'playlist_mincount': 35,
+        'expected_warnings': ['Retrying duplicate page'],
+    }, {
+        'note': 'Filtered paginated list with single page result',
+        'url': 'https://www.youporn.com/porntags/austrian/duration/?min_minutes=10',
+        'info_dict': {
+            'id': 'austrian/duration/min_minutes=10',
+            'title': 'Tag austrian videos by duration (min_minutes=10)',
+        },
+        # number of videos per page is (row x col) 2x3 + 6x4 + 2, or + 3,
+        # or more, varying with number of ads; let's set max as 9x4
+        # NB col 1 may not be shown in non-JS page with site CSS and zoom 100%
+        'playlist_maxcount': 32,
+        'expected_warnings': ['Retrying duplicate page', 'Retrying empty page'],
+    }, {
+        'note': 'Single page of full list',
+        'url': 'https://www.youporn.com/porntags/austrian/?page=1',
+        'info_dict': {
+            'id': 'austrian/page=1',
+            'title': 'Tag austrian videos (page=1)',
+        },
+        'playlist_mincount': 32,
+        'playlist_maxcount': 34,
+        'expected_warnings': ['Retrying duplicate page', 'Retrying empty page'],
+    }]
+
+    # YP tag navigation is broken, loses sort
+    def _get_next_url(self, url, pl_id, html):
+        next_url = super(YouPornTagIE, self)._get_next_url(url, pl_id, html)
+        if next_url:
+            n = self._match_valid_url(next_url)
+            if n:
+                s = n.groupdict().get('sort')
+            if s:
+                u = self._match_valid_url(url)
+                if u:
+                    u = u.groupdict().get('sort')
+                    if s and not u:
+                        n = n.end('sort')
+                        next_url = next_url[:n] + '/' + u + next_url[n:]
+        return next_url
+
+
+class YouPornStarIE(YouPornListBase):
+    IE_DESC = 'YouPorn Pornstar, with description, sorting and pagination'
+    _VALID_URL = r'''(?x)
+        https?://(?:www\.)?youporn\.com/
+        (?P<type>pornstar)/(?P<id>[^/?#&]+)
+        (?:/(?P<sort>rating|views|duration))?/?(?:[#?]|$)
+    '''
+    _PLAYLIST_TITLEBAR_RE = r'^\s*Videos\s+[fF]eaturing\s'
+    _TESTS = [{
+        'note': 'Full list with pagination',
+        'url': 'https://www.youporn.com/pornstar/daynia/',
+        'info_dict': {
+            'id': 'daynia',
+            'title': 'Pornstar Daynia videos',
+            'description': r're:Daynia Rank \d+ Videos \d+ Views [\d,.]+ .+ Subscribers \d+',
+        },
+        'playlist_mincount': 45,
+    }, {
+        'note': 'Single page of full list (no filters here)',
+        'url': 'https://www.youporn.com/pornstar/daynia/?page=1',
+        'info_dict': {
+            'id': 'daynia/page=1',
+            'title': 'Pornstar Daynia videos (page=1)',
+            'description': 're:.{180,}',
+        },
+        'playlist_count': 26,
+    }]
+
+    @staticmethod
+    def _get_title_from_slug(title_slug):
+        return re.sub(r'_', ' ', title_slug).title()
+
+    def _real_extract(self, url):
+        pl_id = self._match_id(url)
+        html = self._download_webpage(url, pl_id)
+        playlist = super(YouPornStarIE, self)._real_extract(url, html=html)
+        INFO_ELEMENT_RE = r'''(?x)
+            <div\s[^>]*\bclass\s*=\s*('|")(?:[\w$-]+\s+|\s)*?pornstar-info-wrapper(?:\s+[\w$-]+|\s)*\1[^>]*>
+            (?P<info>[\s\S]+?)(?:</div>\s*){6,}
+        '''
+
+        infos = self._search_regex(INFO_ELEMENT_RE, html, 'infos', group='info', default='')
+        if infos:
+            infos = re.sub(
+                r'(?:\s*nl=nl)+\s*', ' ',
+                re.sub(r'(?u)\s+', ' ', clean_html(
+                    re.sub('\n', 'nl=nl', infos)))).replace('ribe Subsc', '')
+
+        return merge_dicts({
+            'description': infos.strip() or None,
+        }, playlist)
+
+
+class YouPornVideosIE(YouPornListBase):
+    IE_DESC = 'YouPorn video (browse) playlists, with sorting, filtering and pagination'
+    _VALID_URL = r'''(?x)
+        https?://(?:www\.)?youporn\.com/
+            (?:(?P<id>browse)/)?
+            (?P<sort>(?(id)
+                (?:duration|rating|time|views)|
+                (?:most_(?:favou?rit|view)ed|recommended|top_rated)?))
+            (?:[/#?]|$)
+    '''
+    _PLAYLIST_TITLEBAR_RE = r'\s+(?:[Vv]ideos|VIDEOS)\s*$'
+    _TESTS = [{
+        'note': 'Full list with pagination (too long for test)',
+        'url': 'https://www.youporn.com/',
+        'info_dict': {
+            'id': 'youporn',
+            'title': 'YouPorn videos',
+        },
+        'only_matching': True,
+    }, {
+        'note': 'Full list with pagination (too long for test)',
+        'url': 'https://www.youporn.com/recommended',
+        'info_dict': {
+            'id': 'youporn/recommended',
+            'title': 'YouPorn videos by recommended',
+        },
+        'only_matching': True,
+    }, {
+        'note': 'Full list with pagination (too long for test)',
+        'url': 'https://www.youporn.com/top_rated',
+        'info_dict': {
+            'id': 'youporn/top_rated',
+            'title': 'YouPorn videos by top rated',
+        },
+        'only_matching': True,
+    }, {
+        'note': 'Full list with pagination (too long for test)',
+        'url': 'https://www.youporn.com/browse/time',
+        'info_dict': {
+            'id': 'browse/time',
+            'title': 'YouPorn videos by time',
+        },
+        'only_matching': True,
+    }, {
+        'note': 'Filtered paginated list with single page result',
+        'url': 'https://www.youporn.com/most_favorited/?res=VR&max_minutes=2',
+        'info_dict': {
+            'id': 'youporn/most_favorited/max_minutes=2/res=VR',
+            'title': 'YouPorn videos by most favorited (max_minutes=2,res=VR)',
+        },
+        'playlist_mincount': 10,
+        'playlist_maxcount': 28,
+    }, {
+        'note': 'Filtered paginated list with several pages',
+        'url': 'https://www.youporn.com/most_favorited/?res=VR&max_minutes=5',
+        'info_dict': {
+            'id': 'youporn/most_favorited/max_minutes=5/res=VR',
+            'title': 'YouPorn videos by most favorited (max_minutes=5,res=VR)',
+        },
+        'playlist_mincount': 45,
+    }, {
+        'note': 'Single page of full list',
+        'url': 'https://www.youporn.com/browse/time?page=1',
+        'info_dict': {
+            'id': 'browse/time/page=1',
+            'title': 'YouPorn videos by time (page=1)',
+        },
+        'playlist_count': 36,
+    }]
+
+    @staticmethod
+    def _get_title_from_slug(title_slug):
+        return 'YouPorn' if title_slug == 'browse' else title_slug

From a08f2b7e4567cdc50c0614ee0a4ffdff49b8b6e6 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 15 May 2024 16:57:59 +0100
Subject: [PATCH 752/860] [workflows/ci.yml] Temporary workaround for Python
 3.5 _pip_ failures

https://github.com/actions/setup-python/issues/866
---
 .github/workflows/ci.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 93562afd7..d3b9ae016 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -159,6 +159,9 @@ jobs:
       # wrap broken actions/setup-python@v4
       # NB may run apt-get install in Linux
       uses: ytdl-org/setup-python@v1
+      env:
+        # Temporary workaround for Python 3.5 failures - May 2024
+        PIP_TRUSTED_HOST: "pypi.python.org pypi.org files.pythonhosted.org"
       with:
         python-version: ${{ matrix.python-version }}
         cache-build: true

From 06da64ee51cd405b9392ba484cf7d3d31a88ee30 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 21 Apr 2024 23:42:08 +0100
Subject: [PATCH 753/860] [utils] Update traverse_obj() from yt-dlp * remove
 `is_user_input` option per https://github.com/yt-dlp/yt-dlp/pull/8673 *
 support traversal of compat_xml_etree_ElementTree_Element per
 https://github.com/yt-dlp/yt-dlp/pull/8911 * allow un/branching using all and
 any per https://github.com/yt-dlp/yt-dlp/pull/9571 * support traversal of
 compat_cookies.Morsel and multiple types in `set()` keys per
 https://github.com/yt-dlp/yt-dlp/pull/9577 thx Grub4k for these * also, move
 traversal tests to a separate class * allow for unordered dicts in tests for
 Py<3.7

---
 test/test_utils.py   | 257 +++++++++++++++++++++++++++++++------------
 youtube_dl/compat.py |   9 +-
 youtube_dl/utils.py  | 102 +++++++++++------
 3 files changed, 267 insertions(+), 101 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index ca36909a8..179d21cf5 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -123,6 +123,7 @@ from youtube_dl.compat import (
     compat_chr,
     compat_etree_fromstring,
     compat_getenv,
+    compat_http_cookies,
     compat_os_name,
     compat_setenv,
     compat_str,
@@ -132,10 +133,6 @@ from youtube_dl.compat import (
 
 class TestUtil(unittest.TestCase):
 
-    # yt-dlp shim
-    def assertCountEqual(self, expected, got, msg='count should be the same'):
-        return self.assertEqual(len(tuple(expected)), len(tuple(got)), msg=msg)
-
     def test_timeconvert(self):
         self.assertTrue(timeconvert('') is None)
         self.assertTrue(timeconvert('bougrg') is None)
@@ -740,28 +737,6 @@ class TestUtil(unittest.TestCase):
         self.assertRaises(
             ValueError, multipart_encode, {b'field': b'value'}, boundary='value')
 
-    def test_dict_get(self):
-        FALSE_VALUES = {
-            'none': None,
-            'false': False,
-            'zero': 0,
-            'empty_string': '',
-            'empty_list': [],
-        }
-        d = FALSE_VALUES.copy()
-        d['a'] = 42
-        self.assertEqual(dict_get(d, 'a'), 42)
-        self.assertEqual(dict_get(d, 'b'), None)
-        self.assertEqual(dict_get(d, 'b', 42), 42)
-        self.assertEqual(dict_get(d, ('a', )), 42)
-        self.assertEqual(dict_get(d, ('b', 'a', )), 42)
-        self.assertEqual(dict_get(d, ('b', 'c', 'a', 'd', )), 42)
-        self.assertEqual(dict_get(d, ('b', 'c', )), None)
-        self.assertEqual(dict_get(d, ('b', 'c', ), 42), 42)
-        for key, false_value in FALSE_VALUES.items():
-            self.assertEqual(dict_get(d, ('b', 'c', key, )), None)
-            self.assertEqual(dict_get(d, ('b', 'c', key, ), skip_false_values=False), false_value)
-
     def test_merge_dicts(self):
         self.assertEqual(merge_dicts({'a': 1}, {'b': 2}), {'a': 1, 'b': 2})
         self.assertEqual(merge_dicts({'a': 1}, {'a': 2}), {'a': 1})
@@ -1703,24 +1678,46 @@ Line 1
         self.assertEqual(variadic('spam', allowed_types=dict), 'spam')
         self.assertEqual(variadic('spam', allowed_types=[dict]), 'spam')
 
+    def test_join_nonempty(self):
+        self.assertEqual(join_nonempty('a', 'b'), 'a-b')
+        self.assertEqual(join_nonempty(
+            'a', 'b', 'c', 'd',
+            from_dict={'a': 'c', 'c': [], 'b': 'd', 'd': None}), 'c-d')
+
+
+class TestTraversal(unittest.TestCase):
+    str = compat_str
+    _TEST_DATA = {
+        100: 100,
+        1.2: 1.2,
+        'str': 'str',
+        'None': None,
+        '...': Ellipsis,
+        'urls': [
+            {'index': 0, 'url': 'https://www.example.com/0'},
+            {'index': 1, 'url': 'https://www.example.com/1'},
+        ],
+        'data': (
+            {'index': 2},
+            {'index': 3},
+        ),
+        'dict': {},
+    }
+
+    # yt-dlp shim
+    def assertCountEqual(self, expected, got, msg='count should be the same'):
+        return self.assertEqual(len(tuple(expected)), len(tuple(got)), msg=msg)
+
+    def assertMaybeCountEqual(self, *args, **kwargs):
+        if sys.version_info < (3, 7):
+            # random dict order
+            return self.assertCountEqual(*args, **kwargs)
+        else:
+            return self.assertEqual(*args, **kwargs)
+
     def test_traverse_obj(self):
-        str = compat_str
-        _TEST_DATA = {
-            100: 100,
-            1.2: 1.2,
-            'str': 'str',
-            'None': None,
-            '...': Ellipsis,
-            'urls': [
-                {'index': 0, 'url': 'https://www.example.com/0'},
-                {'index': 1, 'url': 'https://www.example.com/1'},
-            ],
-            'data': (
-                {'index': 2},
-                {'index': 3},
-            ),
-            'dict': {},
-        }
+        str = self.str
+        _TEST_DATA = self._TEST_DATA
 
         # define a pukka Iterable
         def iter_range(stop):
@@ -1771,15 +1768,19 @@ Line 1
         # Test set as key (transformation/type, like `expected_type`)
         self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str.upper), )), ['STR'],
                          msg='Function in set should be a transformation')
+        self.assertEqual(traverse_obj(_TEST_DATA, ('fail', T(lambda _: 'const'))), 'const',
+                         msg='Function in set should always be called')
         self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str))), ['str'],
                          msg='Type in set should be a type filter')
+        self.assertMaybeCountEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str, int))), [100, 'str'],
+                                   msg='Multiple types in set should be a type filter')
         self.assertEqual(traverse_obj(_TEST_DATA, T(dict)), _TEST_DATA,
                          msg='A single set should be wrapped into a path')
         self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str.upper))), ['STR'],
                          msg='Transformation function should not raise')
-        self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str_or_none))),
-                         [item for item in map(str_or_none, _TEST_DATA.values()) if item is not None],
-                         msg='Function in set should be a transformation')
+        self.assertMaybeCountEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str_or_none))),
+                                   [item for item in map(str_or_none, _TEST_DATA.values()) if item is not None],
+                                   msg='Function in set should be a transformation')
         if __debug__:
             with self.assertRaises(Exception, msg='Sets with length != 1 should raise in debug'):
                 traverse_obj(_TEST_DATA, set())
@@ -1992,23 +1993,6 @@ Line 1
         self.assertEqual(traverse_obj({}, (0, slice(1)), _traverse_string=True), [],
                          msg='branching should result in list if `traverse_string`')
 
-        # Test is_user_input behavior
-        _IS_USER_INPUT_DATA = {'range8': list(range(8))}
-        self.assertEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', '3'),
-                                      _is_user_input=True), 3,
-                         msg='allow for string indexing if `is_user_input`')
-        self.assertCountEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', '3:'),
-                                           _is_user_input=True), tuple(range(8))[3:],
-                              msg='allow for string slice if `is_user_input`')
-        self.assertCountEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', ':4:2'),
-                                           _is_user_input=True), tuple(range(8))[:4:2],
-                              msg='allow step in string slice if `is_user_input`')
-        self.assertCountEqual(traverse_obj(_IS_USER_INPUT_DATA, ('range8', ':'),
-                                           _is_user_input=True), range(8),
-                              msg='`:` should be treated as `...` if `is_user_input`')
-        with self.assertRaises(TypeError, msg='too many params should result in error'):
-            traverse_obj(_IS_USER_INPUT_DATA, ('range8', ':::'), _is_user_input=True)
-
         # Test re.Match as input obj
         mobj = re.match(r'^0(12)(?P<group>3)(4)?$', '0123')
         self.assertEqual(traverse_obj(mobj, Ellipsis), [x for x in mobj.groups() if x is not None],
@@ -2030,14 +2014,151 @@ Line 1
         self.assertEqual(traverse_obj(mobj, lambda k, _: k in (0, 'group')), ['0123', '3'],
                          msg='function on a `re.Match` should give group name as well')
 
+        # Test xml.etree.ElementTree.Element as input obj
+        etree = compat_etree_fromstring('''<?xml version="1.0"?>
+        <data>
+            <country name="Liechtenstein">
+                <rank>1</rank>
+                <year>2008</year>
+                <gdppc>141100</gdppc>
+                <neighbor name="Austria" direction="E"/>
+                <neighbor name="Switzerland" direction="W"/>
+            </country>
+            <country name="Singapore">
+                <rank>4</rank>
+                <year>2011</year>
+                <gdppc>59900</gdppc>
+                <neighbor name="Malaysia" direction="N"/>
+            </country>
+            <country name="Panama">
+                <rank>68</rank>
+                <year>2011</year>
+                <gdppc>13600</gdppc>
+                <neighbor name="Costa Rica" direction="W"/>
+                <neighbor name="Colombia" direction="E"/>
+            </country>
+        </data>''')
+        self.assertEqual(traverse_obj(etree, ''), etree,
+                         msg='empty str key should return the element itself')
+        self.assertEqual(traverse_obj(etree, 'country'), list(etree),
+                         msg='str key should return all children with that tag name')
+        self.assertEqual(traverse_obj(etree, Ellipsis), list(etree),
+                         msg='`...` as key should return all children')
+        self.assertEqual(traverse_obj(etree, lambda _, x: x[0].text == '4'), [etree[1]],
+                         msg='function as key should get element as value')
+        self.assertEqual(traverse_obj(etree, lambda i, _: i == 1), [etree[1]],
+                         msg='function as key should get index as key')
+        self.assertEqual(traverse_obj(etree, 0), etree[0],
+                         msg='int key should return the nth child')
+        self.assertEqual(traverse_obj(etree, './/neighbor/@name'),
+                         ['Austria', 'Switzerland', 'Malaysia', 'Costa Rica', 'Colombia'],
+                         msg='`@<attribute>` at end of path should give that attribute')
+        self.assertEqual(traverse_obj(etree, '//neighbor/@fail'), [None, None, None, None, None],
+                         msg='`@<nonexistent>` at end of path should give `None`')
+        self.assertEqual(traverse_obj(etree, ('//neighbor/@', 2)), {'name': 'Malaysia', 'direction': 'N'},
+                         msg='`@` should give the full attribute dict')
+        self.assertEqual(traverse_obj(etree, '//year/text()'), ['2008', '2011', '2011'],
+                         msg='`text()` at end of path should give the inner text')
+        self.assertEqual(traverse_obj(etree, '//*[@direction]/@direction'), ['E', 'W', 'N', 'W', 'E'],
+                         msg='full python xpath features should be supported')
+        self.assertEqual(traverse_obj(etree, (0, '@name')), 'Liechtenstein',
+                         msg='special transformations should act on current element')
+        self.assertEqual(traverse_obj(etree, ('country', 0, Ellipsis, 'text()', T(int_or_none))), [1, 2008, 141100],
+                         msg='special transformations should act on current element')
+
+    def test_traversal_unbranching(self):
+        # str = self.str
+        _TEST_DATA = self._TEST_DATA
+
+        self.assertEqual(traverse_obj(_TEST_DATA, [(100, 1.2), all]), [100, 1.2],
+                         msg='`all` should give all results as list')
+        self.assertEqual(traverse_obj(_TEST_DATA, [(100, 1.2), any]), 100,
+                         msg='`any` should give the first result')
+        self.assertEqual(traverse_obj(_TEST_DATA, [100, all]), [100],
+                         msg='`all` should give list if non branching')
+        self.assertEqual(traverse_obj(_TEST_DATA, [100, any]), 100,
+                         msg='`any` should give single item if non branching')
+        self.assertEqual(traverse_obj(_TEST_DATA, [('dict', 'None', 100), all]), [100],
+                         msg='`all` should filter `None` and empty dict')
+        self.assertEqual(traverse_obj(_TEST_DATA, [('dict', 'None', 100), any]), 100,
+                         msg='`any` should filter `None` and empty dict')
+        self.assertEqual(traverse_obj(_TEST_DATA, [{
+            'all': [('dict', 'None', 100, 1.2), all],
+            'any': [('dict', 'None', 100, 1.2), any],
+        }]), {'all': [100, 1.2], 'any': 100},
+            msg='`all`/`any` should apply to each dict path separately')
+        self.assertEqual(traverse_obj(_TEST_DATA, [{
+            'all': [('dict', 'None', 100, 1.2), all],
+            'any': [('dict', 'None', 100, 1.2), any],
+        }], get_all=False), {'all': [100, 1.2], 'any': 100},
+            msg='`all`/`any` should apply to dict regardless of `get_all`')
+        self.assertIs(traverse_obj(_TEST_DATA, [('dict', 'None', 100, 1.2), all, T(float)]), None,
+                      msg='`all` should reset branching status')
+        self.assertIs(traverse_obj(_TEST_DATA, [('dict', 'None', 100, 1.2), any, T(float)]), None,
+                      msg='`any` should reset branching status')
+        self.assertEqual(traverse_obj(_TEST_DATA, [('dict', 'None', 100, 1.2), all, Ellipsis, T(float)]), [1.2],
+                         msg='`all` should allow further branching')
+        self.assertEqual(traverse_obj(_TEST_DATA, [('dict', 'None', 'urls', 'data'), any, Ellipsis, 'index']), [0, 1],
+                         msg='`any` should allow further branching')
+
+    def test_traversal_morsel(self):
+        values = {
+            'expires': 'a',
+            'path': 'b',
+            'comment': 'c',
+            'domain': 'd',
+            'max-age': 'e',
+            'secure': 'f',
+            'httponly': 'g',
+            'version': 'h',
+            'samesite': 'i',
+        }
+        # SameSite added in Py3.8, breaks .update for 3.5-3.7
+        if sys.version_info < (3, 8):
+            del values['samesite']
+        morsel = compat_http_cookies.Morsel()
+        morsel.set(str('item_key'), 'item_value', 'coded_value')
+        morsel.update(values)
+        values['key'] = str('item_key')
+        values['value'] = 'item_value'
+        values = dict((str(k), v) for k, v in values.items())
+        # make test pass even without ordered dict
+        value_set = set(values.values())
+
+        for key, value in values.items():
+            self.assertEqual(traverse_obj(morsel, key), value,
+                             msg='Morsel should provide access to all values')
+        self.assertEqual(set(traverse_obj(morsel, Ellipsis)), value_set,
+                         msg='`...` should yield all values')
+        self.assertEqual(set(traverse_obj(morsel, lambda k, v: True)), value_set,
+                         msg='function key should yield all values')
+        self.assertIs(traverse_obj(morsel, [(None,), any]), morsel,
+                      msg='Morsel should not be implicitly changed to dict on usage')
+
     def test_get_first(self):
         self.assertEqual(get_first([{'a': None}, {'a': 'spam'}], 'a'), 'spam')
 
-    def test_join_nonempty(self):
-        self.assertEqual(join_nonempty('a', 'b'), 'a-b')
-        self.assertEqual(join_nonempty(
-            'a', 'b', 'c', 'd',
-            from_dict={'a': 'c', 'c': [], 'b': 'd', 'd': None}), 'c-d')
+    def test_dict_get(self):
+        FALSE_VALUES = {
+            'none': None,
+            'false': False,
+            'zero': 0,
+            'empty_string': '',
+            'empty_list': [],
+        }
+        d = FALSE_VALUES.copy()
+        d['a'] = 42
+        self.assertEqual(dict_get(d, 'a'), 42)
+        self.assertEqual(dict_get(d, 'b'), None)
+        self.assertEqual(dict_get(d, 'b', 42), 42)
+        self.assertEqual(dict_get(d, ('a', )), 42)
+        self.assertEqual(dict_get(d, ('b', 'a', )), 42)
+        self.assertEqual(dict_get(d, ('b', 'c', 'a', 'd', )), 42)
+        self.assertEqual(dict_get(d, ('b', 'c', )), None)
+        self.assertEqual(dict_get(d, ('b', 'c', ), 42), 42)
+        for key, false_value in FALSE_VALUES.items():
+            self.assertEqual(dict_get(d, ('b', 'c', key, )), None)
+            self.assertEqual(dict_get(d, ('b', 'c', key, ), skip_false_values=False), false_value)
 
 
 if __name__ == '__main__':
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 53ff2a892..d5485c7e8 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2719,8 +2719,14 @@ if sys.version_info < (2, 7):
         if isinstance(xpath, compat_str):
             xpath = xpath.encode('ascii')
         return xpath
+
+    def compat_etree_iterfind(element, match):
+        for from_ in element.findall(match):
+            yield from_
+
 else:
     compat_xpath = lambda xpath: xpath
+    compat_etree_iterfind = lambda element, match: element.iterfind(match)
 
 
 compat_os_name = os._name if os.name == 'java' else os.name
@@ -2955,7 +2961,7 @@ except ImportError:
             return self
 
         def __exit__(self, exc_type, exc_val, exc_tb):
-            return exc_val is not None and isinstance(exc_val, self._exceptions or tuple())
+            return exc_type is not None and issubclass(exc_type, self._exceptions or tuple())
 
 
 # subprocess.Popen context manager
@@ -3308,6 +3314,7 @@ __all__ = [
     'compat_contextlib_suppress',
     'compat_ctypes_WINFUNCTYPE',
     'compat_etree_fromstring',
+    'compat_etree_iterfind',
     'compat_filter',
     'compat_get_terminal_size',
     'compat_getenv',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index e1b05b307..cd4303566 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -49,11 +49,14 @@ from .compat import (
     compat_cookiejar,
     compat_ctypes_WINFUNCTYPE,
     compat_datetime_timedelta_total_seconds,
+    compat_etree_Element,
     compat_etree_fromstring,
+    compat_etree_iterfind,
     compat_expanduser,
     compat_html_entities,
     compat_html_entities_html5,
     compat_http_client,
+    compat_http_cookies,
     compat_integer_types,
     compat_kwargs,
     compat_ncompress as ncompress,
@@ -6253,15 +6256,16 @@ if __debug__:
 
 def traverse_obj(obj, *paths, **kwargs):
     """
-    Safely traverse nested `dict`s and `Iterable`s
+    Safely traverse nested `dict`s and `Iterable`s, etc
 
     >>> obj = [{}, {"key": "value"}]
     >>> traverse_obj(obj, (1, "key"))
-    "value"
+    'value'
 
     Each of the provided `paths` is tested and the first producing a valid result will be returned.
     The next path will also be tested if the path branched but no results could be found.
-    Supported values for traversal are `Mapping`, `Iterable` and `re.Match`.
+    Supported values for traversal are `Mapping`, `Iterable`, `re.Match`, `xml.etree.ElementTree`
+    (xpath) and `http.cookies.Morsel`.
     Unhelpful values (`{}`, `None`) are treated as the absence of a value and discarded.
 
     The paths will be wrapped in `variadic`, so that `'key'` is conveniently the same as `('key', )`.
@@ -6269,8 +6273,9 @@ def traverse_obj(obj, *paths, **kwargs):
     The keys in the path can be one of:
         - `None`:           Return the current object.
         - `set`:            Requires the only item in the set to be a type or function,
-                            like `{type}`/`{func}`. If a `type`, returns only values
-                            of this type. If a function, returns `func(obj)`.
+                            like `{type}`/`{type, type, ...}`/`{func}`. If one or more `type`s,
+                            return only values that have one of the types. If a function,
+                            return `func(obj)`.
         - `str`/`int`:      Return `obj[key]`. For `re.Match`, return `obj.group(key)`.
         - `slice`:          Branch out and return all values in `obj[key]`.
         - `Ellipsis`:       Branch out and return a list of all values.
@@ -6282,8 +6287,10 @@ def traverse_obj(obj, *paths, **kwargs):
                             For `Iterable`s, `key` is the enumeration count of the value.
                             For `re.Match`es, `key` is the group number (0 = full match)
                             as well as additionally any group names, if given.
-        - `dict`            Transform the current object and return a matching dict.
+        - `dict`:           Transform the current object and return a matching dict.
                             Read as: `{key: traverse_obj(obj, path) for key, path in dct.items()}`.
+        - `any`-builtin:    Take the first matching object and return it, resetting branching.
+        - `all`-builtin:    Take all matching objects and return them as a list, resetting branching.
 
         `tuple`, `list`, and `dict` all support nested paths and branches.
 
@@ -6299,10 +6306,8 @@ def traverse_obj(obj, *paths, **kwargs):
     @param get_all          If `False`, return the first matching result, otherwise all matching ones.
     @param casesense        If `False`, consider string dictionary keys as case insensitive.
 
-    The following are only meant to be used by YoutubeDL.prepare_outtmpl and are not part of the API
+    The following is only meant to be used by YoutubeDL.prepare_outtmpl and is not part of the API
 
-    @param _is_user_input    Whether the keys are generated from user input.
-                            If `True` strings get converted to `int`/`slice` if needed.
     @param _traverse_string  Whether to traverse into objects as strings.
                             If `True`, any non-compatible object will first be
                             converted into a string and then traversed into.
@@ -6322,7 +6327,6 @@ def traverse_obj(obj, *paths, **kwargs):
     expected_type = kwargs.get('expected_type')
     get_all = kwargs.get('get_all', True)
     casesense = kwargs.get('casesense', True)
-    _is_user_input = kwargs.get('_is_user_input', False)
     _traverse_string = kwargs.get('_traverse_string', False)
 
     # instant compat
@@ -6336,10 +6340,8 @@ def traverse_obj(obj, *paths, **kwargs):
         type_test = lambda val: try_call(expected_type or IDENTITY, args=(val,))
 
     def lookup_or_none(v, k, getter=None):
-        try:
+        with compat_contextlib_suppress(LookupError):
             return getter(v, k) if getter else v[k]
-        except IndexError:
-            return None
 
     def from_iterable(iterables):
         # chain.from_iterable(['ABC', 'DEF']) --> A B C D E F
@@ -6361,12 +6363,13 @@ def traverse_obj(obj, *paths, **kwargs):
             result = obj
 
         elif isinstance(key, set):
-            assert len(key) == 1, 'Set should only be used to wrap a single item'
-            item = next(iter(key))
-            if isinstance(item, type):
-                result = obj if isinstance(obj, item) else None
+            assert len(key) >= 1, 'At least one item is required in a `set` key'
+            if all(isinstance(item, type) for item in key):
+                result = obj if isinstance(obj, tuple(key)) else None
             else:
-                result = try_call(item, args=(obj,))
+                item = next(iter(key))
+                assert len(key) == 1, 'Multiple items in a `set` key must all be types'
+                result = try_call(item, args=(obj,)) if not isinstance(item, type) else None
 
         elif isinstance(key, (list, tuple)):
             branching = True
@@ -6375,9 +6378,11 @@ def traverse_obj(obj, *paths, **kwargs):
 
         elif key is Ellipsis:
             branching = True
+            if isinstance(obj, compat_http_cookies.Morsel):
+                obj = dict(obj, key=obj.key, value=obj.value)
             if isinstance(obj, compat_collections_abc.Mapping):
                 result = obj.values()
-            elif is_iterable_like(obj):
+            elif is_iterable_like(obj, (compat_collections_abc.Iterable, compat_etree_Element)):
                 result = obj
             elif isinstance(obj, compat_re_Match):
                 result = obj.groups()
@@ -6389,9 +6394,11 @@ def traverse_obj(obj, *paths, **kwargs):
 
         elif callable(key):
             branching = True
+            if isinstance(obj, compat_http_cookies.Morsel):
+                obj = dict(obj, key=obj.key, value=obj.value)
             if isinstance(obj, compat_collections_abc.Mapping):
                 iter_obj = obj.items()
-            elif is_iterable_like(obj):
+            elif is_iterable_like(obj, (compat_collections_abc.Iterable, compat_etree_Element)):
                 iter_obj = enumerate(obj)
             elif isinstance(obj, compat_re_Match):
                 iter_obj = itertools.chain(
@@ -6413,6 +6420,8 @@ def traverse_obj(obj, *paths, **kwargs):
                           if v is not None or default is not NO_DEFAULT) or None
 
         elif isinstance(obj, compat_collections_abc.Mapping):
+            if isinstance(obj, compat_http_cookies.Morsel):
+                obj = dict(obj, key=obj.key, value=obj.value)
             result = (try_call(obj.get, args=(key,))
                       if casesense or try_call(obj.__contains__, args=(key,))
                       else next((v for k, v in obj.items() if casefold(k) == key), None))
@@ -6430,12 +6439,40 @@ def traverse_obj(obj, *paths, **kwargs):
         else:
             result = None
             if isinstance(key, (int, slice)):
-                if is_iterable_like(obj, compat_collections_abc.Sequence):
+                if is_iterable_like(obj, (compat_collections_abc.Sequence, compat_etree_Element)):
                     branching = isinstance(key, slice)
                     result = lookup_or_none(obj, key)
                 elif _traverse_string:
                     result = lookup_or_none(str(obj), key)
 
+            elif isinstance(obj, compat_etree_Element) and isinstance(key, str):
+                xpath, _, special = key.rpartition('/')
+                if not special.startswith('@') and not special.endswith('()'):
+                    xpath = key
+                    special = None
+
+                # Allow abbreviations of relative paths, absolute paths error
+                if xpath.startswith('/'):
+                    xpath = '.' + xpath
+                elif xpath and not xpath.startswith('./'):
+                    xpath = './' + xpath
+
+                def apply_specials(element):
+                    if special is None:
+                        return element
+                    if special == '@':
+                        return element.attrib
+                    if special.startswith('@'):
+                        return try_call(element.attrib.get, args=(special[1:],))
+                    if special == 'text()':
+                        return element.text
+                    raise SyntaxError('apply_specials is missing case for {0!r}'.format(special))
+
+                if xpath:
+                    result = list(map(apply_specials, compat_etree_iterfind(obj, xpath)))
+                else:
+                    result = apply_specials(obj)
+
         return branching, result if branching else (result,)
 
     def lazy_last(iterable):
@@ -6456,17 +6493,18 @@ def traverse_obj(obj, *paths, **kwargs):
 
         key = None
         for last, key in lazy_last(variadic(path, (str, bytes, dict, set))):
-            if _is_user_input and isinstance(key, str):
-                if key == ':':
-                    key = Ellipsis
-                elif ':' in key:
-                    key = slice(*map(int_or_none, key.split(':')))
-                elif int_or_none(key) is not None:
-                    key = int(key)
-
             if not casesense and isinstance(key, str):
                 key = compat_casefold(key)
 
+            if key in (any, all):
+                has_branched = False
+                filtered_objs = (obj for obj in objs if obj not in (None, {}))
+                if key is any:
+                    objs = (next(filtered_objs, None),)
+                else:
+                    objs = (list(filtered_objs),)
+                continue
+
             if __debug__ and callable(key):
                 # Verify function signature
                 _try_bind_args(key, None, None)
@@ -6505,9 +6543,9 @@ def traverse_obj(obj, *paths, **kwargs):
     return None if default is NO_DEFAULT else default
 
 
-def T(x):
-    """ For use in yt-dl instead of {type} or set((type,)) """
-    return set((x,))
+def T(*x):
+    """ For use in yt-dl instead of {type, ...} or set((type, ...)) """
+    return set(x)
 
 
 def get_first(obj, keys, **kwargs):

From 34484e49f5cd91a830f5459a5b673b7c05a22e24 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 28 May 2024 16:38:20 +0100
Subject: [PATCH 754/860] [compat] Improve compat_etree_iterfind for Py2.6
 Adapted from
 https://raw.githubusercontent.com/python/cpython/2.7/Lib/xml/etree/ElementPath.py

---
 youtube_dl/compat.py | 214 ++++++++++++++++++++++++++++++++++++++++++-
 1 file changed, 211 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index d5485c7e8..0371896ab 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2720,9 +2720,217 @@ if sys.version_info < (2, 7):
             xpath = xpath.encode('ascii')
         return xpath
 
-    def compat_etree_iterfind(element, match):
-        for from_ in element.findall(match):
-            yield from_
+    # further code below based on CPython 2.7 source
+    import functools
+
+    _xpath_tokenizer_re = re.compile(r'''(?x)
+        (                                   # (1)
+            '[^']*'|"[^"]*"|                # quoted strings, or
+            ::|//?|\.\.|\(\)|[/.*:[\]()@=]  # navigation specials
+        )|                                  # or (2)
+        ((?:\{[^}]+\})?[^/[\]()@=\s]+)|     # token: optional {ns}, no specials
+        \s+                                 # or white space
+    ''')
+
+    def _xpath_tokenizer(pattern, namespaces=None):
+        for token in _xpath_tokenizer_re.findall(pattern):
+            tag = token[1]
+            if tag and tag[0] != "{" and ":" in tag:
+                try:
+                    if not namespaces:
+                        raise KeyError
+                    prefix, uri = tag.split(":", 1)
+                    yield token[0], "{%s}%s" % (namespaces[prefix], uri)
+                except KeyError:
+                    raise SyntaxError("prefix %r not found in prefix map" % prefix)
+            else:
+                yield token
+
+    def _get_parent_map(context):
+        parent_map = context.parent_map
+        if parent_map is None:
+            context.parent_map = parent_map = {}
+            for p in context.root.getiterator():
+                for e in p:
+                    parent_map[e] = p
+        return parent_map
+
+    def _select(context, result, filter_fn=lambda *_: True):
+        for elem in result:
+            for e in elem:
+                if filter_fn(e, elem):
+                    yield e
+
+    def _prepare_child(next_, token):
+        tag = token[1]
+        return functools.partial(_select, filter_fn=lambda e, _: e.tag == tag)
+
+    def _prepare_star(next_, token):
+        return _select
+
+    def _prepare_self(next_, token):
+        return lambda _, result: (e for e in result)
+
+    def _prepare_descendant(next_, token):
+        token = next(next_)
+        if token[0] == "*":
+            tag = "*"
+        elif not token[0]:
+            tag = token[1]
+        else:
+            raise SyntaxError("invalid descendant")
+
+        def select(context, result):
+            for elem in result:
+                for e in elem.getiterator(tag):
+                    if e is not elem:
+                        yield e
+        return select
+
+    def _prepare_parent(next_, token):
+        def select(context, result):
+            # FIXME: raise error if .. is applied at toplevel?
+            parent_map = _get_parent_map(context)
+            result_map = {}
+            for elem in result:
+                if elem in parent_map:
+                    parent = parent_map[elem]
+                    if parent not in result_map:
+                        result_map[parent] = None
+                        yield parent
+        return select
+
+    def _prepare_predicate(next_, token):
+        signature = []
+        predicate = []
+        for token in next_:
+            if token[0] == "]":
+                break
+            if token[0] and token[0][:1] in "'\"":
+                token = "'", token[0][1:-1]
+            signature.append(token[0] or "-")
+            predicate.append(token[1])
+
+        def select(context, result, filter_fn=lambda _: True):
+            for elem in result:
+                if filter_fn(elem):
+                    yield elem
+
+        signature = "".join(signature)
+        # use signature to determine predicate type
+        if signature == "@-":
+            # [@attribute] predicate
+            key = predicate[1]
+            return functools.partial(
+                select, filter_fn=lambda el: el.get(key) is not None)
+        if signature == "@-='":
+            # [@attribute='value']
+            key = predicate[1]
+            value = predicate[-1]
+            return functools.partial(
+                select, filter_fn=lambda el: el.get(key) == value)
+        if signature == "-" and not re.match(r"\d+$", predicate[0]):
+            # [tag]
+            tag = predicate[0]
+            return functools.partial(
+                select, filter_fn=lambda el: el.find(tag) is not None)
+        if signature == "-='" and not re.match(r"\d+$", predicate[0]):
+            # [tag='value']
+            tag = predicate[0]
+            value = predicate[-1]
+
+            def itertext(el):
+                for e in el.getiterator():
+                    e = e.text
+                    if e:
+                        yield e
+
+            def select(context, result):
+                for elem in result:
+                    for e in elem.findall(tag):
+                        if "".join(itertext(e)) == value:
+                            yield elem
+                            break
+            return select
+        if signature == "-" or signature == "-()" or signature == "-()-":
+            # [index] or [last()] or [last()-index]
+            if signature == "-":
+                index = int(predicate[0]) - 1
+            else:
+                if predicate[0] != "last":
+                    raise SyntaxError("unsupported function")
+                if signature == "-()-":
+                    try:
+                        index = int(predicate[2]) - 1
+                    except ValueError:
+                        raise SyntaxError("unsupported expression")
+                else:
+                    index = -1
+
+            def select(context, result):
+                parent_map = _get_parent_map(context)
+                for elem in result:
+                    try:
+                        parent = parent_map[elem]
+                        # FIXME: what if the selector is "*" ?
+                        elems = list(parent.findall(elem.tag))
+                        if elems[index] is elem:
+                            yield elem
+                    except (IndexError, KeyError):
+                        pass
+            return select
+        raise SyntaxError("invalid predicate")
+
+    ops = {
+        "": _prepare_child,
+        "*": _prepare_star,
+        ".": _prepare_self,
+        "..": _prepare_parent,
+        "//": _prepare_descendant,
+        "[": _prepare_predicate,
+    }
+
+    _cache = {}
+
+    class _SelectorContext:
+        parent_map = None
+
+        def __init__(self, root):
+            self.root = root
+
+    ##
+    # Generate all matching objects.
+
+    def compat_etree_iterfind(elem, path, namespaces=None):
+        # compile selector pattern
+        if path[-1:] == "/":
+            path = path + "*"  # implicit all (FIXME: keep this?)
+        try:
+            selector = _cache[path]
+        except KeyError:
+            if len(_cache) > 100:
+                _cache.clear()
+            if path[:1] == "/":
+                raise SyntaxError("cannot use absolute path on element")
+            tokens = _xpath_tokenizer(path, namespaces)
+            selector = []
+            for token in tokens:
+                if token[0] == "/":
+                    continue
+                try:
+                    selector.append(ops[token[0]](tokens, token))
+                except StopIteration:
+                    raise SyntaxError("invalid path")
+            _cache[path] = selector
+        # execute selector pattern
+        result = [elem]
+        context = _SelectorContext(elem)
+        for select in selector:
+            result = select(context, result)
+        return result
+
+    # end of code based on CPython 2.7 source
+
 
 else:
     compat_xpath = lambda xpath: xpath

From eee9a247eb3ef876ce6d9f5e34275e46a2d06d10 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 28 May 2024 17:16:58 +0100
Subject: [PATCH 755/860] [utils] Split out traversal.py dummy and traversal
 tests

---
 test/test_traversal.py  | 509 ++++++++++++++++++++++++++++++++++++++++
 test/test_utils.py      | 483 --------------------------------------
 youtube_dl/traversal.py |  10 +
 3 files changed, 519 insertions(+), 483 deletions(-)
 create mode 100644 test/test_traversal.py
 create mode 100644 youtube_dl/traversal.py

diff --git a/test/test_traversal.py b/test/test_traversal.py
new file mode 100644
index 000000000..00a428edb
--- /dev/null
+++ b/test/test_traversal.py
@@ -0,0 +1,509 @@
+#!/usr/bin/env python
+# coding: utf-8
+from __future__ import unicode_literals
+
+# Allow direct execution
+import os
+import sys
+import unittest
+sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
+
+
+import re
+
+from youtube_dl.traversal import (
+    dict_get,
+    get_first,
+    T,
+    traverse_obj,
+)
+from youtube_dl.compat import (
+    compat_etree_fromstring,
+    compat_http_cookies,
+    compat_str,
+)
+from youtube_dl.utils import (
+    int_or_none,
+    str_or_none,
+)
+
+_TEST_DATA = {
+    100: 100,
+    1.2: 1.2,
+    'str': 'str',
+    'None': None,
+    '...': Ellipsis,
+    'urls': [
+        {'index': 0, 'url': 'https://www.example.com/0'},
+        {'index': 1, 'url': 'https://www.example.com/1'},
+    ],
+    'data': (
+        {'index': 2},
+        {'index': 3},
+    ),
+    'dict': {},
+}
+
+
+if sys.version_info < (3, 0):
+    class _TestCase(unittest.TestCase):
+
+        def assertCountEqual(self, *args, **kwargs):
+            return self.assertItemsEqual(*args, **kwargs)
+else:
+    _TestCase = unittest.TestCase
+
+
+class TestTraversal(_TestCase):
+    def assertMaybeCountEqual(self, *args, **kwargs):
+        if sys.version_info < (3, 7):
+            # random dict order
+            return self.assertCountEqual(*args, **kwargs)
+        else:
+            return self.assertEqual(*args, **kwargs)
+
+    def test_traverse_obj(self):
+        # instant compat
+        str = compat_str
+
+        # define a pukka Iterable
+        def iter_range(stop):
+            for from_ in range(stop):
+                yield from_
+
+        # Test base functionality
+        self.assertEqual(traverse_obj(_TEST_DATA, ('str',)), 'str',
+                         msg='allow tuple path')
+        self.assertEqual(traverse_obj(_TEST_DATA, ['str']), 'str',
+                         msg='allow list path')
+        self.assertEqual(traverse_obj(_TEST_DATA, (value for value in ("str",))), 'str',
+                         msg='allow iterable path')
+        self.assertEqual(traverse_obj(_TEST_DATA, 'str'), 'str',
+                         msg='single items should be treated as a path')
+        self.assertEqual(traverse_obj(_TEST_DATA, None), _TEST_DATA)
+        self.assertEqual(traverse_obj(_TEST_DATA, 100), 100)
+        self.assertEqual(traverse_obj(_TEST_DATA, 1.2), 1.2)
+
+        # Test Ellipsis behavior
+        self.assertCountEqual(traverse_obj(_TEST_DATA, Ellipsis),
+                              (item for item in _TEST_DATA.values() if item not in (None, {})),
+                              msg='`...` should give all non-discarded values')
+        self.assertCountEqual(traverse_obj(_TEST_DATA, ('urls', 0, Ellipsis)), _TEST_DATA['urls'][0].values(),
+                              msg='`...` selection for dicts should select all values')
+        self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, Ellipsis, 'url')),
+                         ['https://www.example.com/0', 'https://www.example.com/1'],
+                         msg='nested `...` queries should work')
+        self.assertCountEqual(traverse_obj(_TEST_DATA, (Ellipsis, Ellipsis, 'index')), iter_range(4),
+                              msg='`...` query result should be flattened')
+        self.assertEqual(traverse_obj(iter(range(4)), Ellipsis), list(range(4)),
+                         msg='`...` should accept iterables')
+
+        # Test function as key
+        self.assertEqual(traverse_obj(_TEST_DATA, lambda x, y: x == 'urls' and isinstance(y, list)),
+                         [_TEST_DATA['urls']],
+                         msg='function as query key should perform a filter based on (key, value)')
+        self.assertCountEqual(traverse_obj(_TEST_DATA, lambda _, x: isinstance(x[0], str)), set(('str',)),
+                              msg='exceptions in the query function should be caught')
+        self.assertEqual(traverse_obj(iter(range(4)), lambda _, x: x % 2 == 0), [0, 2],
+                         msg='function key should accept iterables')
+        if __debug__:
+            with self.assertRaises(Exception, msg='Wrong function signature should raise in debug'):
+                traverse_obj(_TEST_DATA, lambda a: Ellipsis)
+            with self.assertRaises(Exception, msg='Wrong function signature should raise in debug'):
+                traverse_obj(_TEST_DATA, lambda a, b, c: Ellipsis)
+
+        # Test set as key (transformation/type, like `expected_type`)
+        self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str.upper), )), ['STR'],
+                         msg='Function in set should be a transformation')
+        self.assertEqual(traverse_obj(_TEST_DATA, ('fail', T(lambda _: 'const'))), 'const',
+                         msg='Function in set should always be called')
+        self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str))), ['str'],
+                         msg='Type in set should be a type filter')
+        self.assertMaybeCountEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str, int))), [100, 'str'],
+                                   msg='Multiple types in set should be a type filter')
+        self.assertEqual(traverse_obj(_TEST_DATA, T(dict)), _TEST_DATA,
+                         msg='A single set should be wrapped into a path')
+        self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str.upper))), ['STR'],
+                         msg='Transformation function should not raise')
+        self.assertMaybeCountEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str_or_none))),
+                                   [item for item in map(str_or_none, _TEST_DATA.values()) if item is not None],
+                                   msg='Function in set should be a transformation')
+        if __debug__:
+            with self.assertRaises(Exception, msg='Sets with length != 1 should raise in debug'):
+                traverse_obj(_TEST_DATA, set())
+            with self.assertRaises(Exception, msg='Sets with length != 1 should raise in debug'):
+                traverse_obj(_TEST_DATA, set((str.upper, str)))
+
+        # Test `slice` as a key
+        _SLICE_DATA = [0, 1, 2, 3, 4]
+        self.assertEqual(traverse_obj(_TEST_DATA, ('dict', slice(1))), None,
+                         msg='slice on a dictionary should not throw')
+        self.assertEqual(traverse_obj(_SLICE_DATA, slice(1)), _SLICE_DATA[:1],
+                         msg='slice key should apply slice to sequence')
+        self.assertEqual(traverse_obj(_SLICE_DATA, slice(1, 2)), _SLICE_DATA[1:2],
+                         msg='slice key should apply slice to sequence')
+        self.assertEqual(traverse_obj(_SLICE_DATA, slice(1, 4, 2)), _SLICE_DATA[1:4:2],
+                         msg='slice key should apply slice to sequence')
+
+        # Test alternative paths
+        self.assertEqual(traverse_obj(_TEST_DATA, 'fail', 'str'), 'str',
+                         msg='multiple `paths` should be treated as alternative paths')
+        self.assertEqual(traverse_obj(_TEST_DATA, 'str', 100), 'str',
+                         msg='alternatives should exit early')
+        self.assertEqual(traverse_obj(_TEST_DATA, 'fail', 'fail'), None,
+                         msg='alternatives should return `default` if exhausted')
+        self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, 'fail'), 100), 100,
+                         msg='alternatives should track their own branching return')
+        self.assertEqual(traverse_obj(_TEST_DATA, ('dict', Ellipsis), ('data', Ellipsis)), list(_TEST_DATA['data']),
+                         msg='alternatives on empty objects should search further')
+
+        # Test branch and path nesting
+        self.assertEqual(traverse_obj(_TEST_DATA, ('urls', (3, 0), 'url')), ['https://www.example.com/0'],
+                         msg='tuple as key should be treated as branches')
+        self.assertEqual(traverse_obj(_TEST_DATA, ('urls', [3, 0], 'url')), ['https://www.example.com/0'],
+                         msg='list as key should be treated as branches')
+        self.assertEqual(traverse_obj(_TEST_DATA, ('urls', ((1, 'fail'), (0, 'url')))), ['https://www.example.com/0'],
+                         msg='double nesting in path should be treated as paths')
+        self.assertEqual(traverse_obj(['0', [1, 2]], [(0, 1), 0]), [1],
+                         msg='do not fail early on branching')
+        self.assertCountEqual(traverse_obj(_TEST_DATA, ('urls', ((1, ('fail', 'url')), (0, 'url')))),
+                              ['https://www.example.com/0', 'https://www.example.com/1'],
+                              msg='triple nesting in path should be treated as branches')
+        self.assertEqual(traverse_obj(_TEST_DATA, ('urls', ('fail', (Ellipsis, 'url')))),
+                         ['https://www.example.com/0', 'https://www.example.com/1'],
+                         msg='ellipsis as branch path start gets flattened')
+
+        # Test dictionary as key
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2}), {0: 100, 1: 1.2},
+                         msg='dict key should result in a dict with the same keys')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', 0, 'url')}),
+                         {0: 'https://www.example.com/0'},
+                         msg='dict key should allow paths')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', (3, 0), 'url')}),
+                         {0: ['https://www.example.com/0']},
+                         msg='tuple in dict path should be treated as branches')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', ((1, 'fail'), (0, 'url')))}),
+                         {0: ['https://www.example.com/0']},
+                         msg='double nesting in dict path should be treated as paths')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', ((1, ('fail', 'url')), (0, 'url')))}),
+                         {0: ['https://www.example.com/1', 'https://www.example.com/0']},
+                         msg='triple nesting in dict path should be treated as branches')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'fail'}), {},
+                         msg='remove `None` values when top level dict key fails')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'fail'}, default=Ellipsis), {0: Ellipsis},
+                         msg='use `default` if key fails and `default`')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}), {},
+                         msg='remove empty values when dict key')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}, default=Ellipsis), {0: Ellipsis},
+                         msg='use `default` when dict key and a default')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 'fail'}}), {},
+                         msg='remove empty values when nested dict key fails')
+        self.assertEqual(traverse_obj(None, {0: 'fail'}), {},
+                         msg='default to dict if pruned')
+        self.assertEqual(traverse_obj(None, {0: 'fail'}, default=Ellipsis), {0: Ellipsis},
+                         msg='default to dict if pruned and default is given')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 'fail'}}, default=Ellipsis), {0: {0: Ellipsis}},
+                         msg='use nested `default` when nested dict key fails and `default`')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('dict', Ellipsis)}), {},
+                         msg='remove key if branch in dict key not successful')
+
+        # Testing default parameter behavior
+        _DEFAULT_DATA = {'None': None, 'int': 0, 'list': []}
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, 'fail'), None,
+                         msg='default value should be `None`')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, 'fail', 'fail', default=Ellipsis), Ellipsis,
+                         msg='chained fails should result in default')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, 'None', 'int'), 0,
+                         msg='should not short cirquit on `None`')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, 'fail', default=1), 1,
+                         msg='invalid dict key should result in `default`')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, 'None', default=1), 1,
+                         msg='`None` is a deliberate sentinel and should become `default`')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, ('list', 10)), None,
+                         msg='`IndexError` should result in `default`')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, (Ellipsis, 'fail'), default=1), 1,
+                         msg='if branched but not successful return `default` if defined, not `[]`')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, (Ellipsis, 'fail'), default=None), None,
+                         msg='if branched but not successful return `default` even if `default` is `None`')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, (Ellipsis, 'fail')), [],
+                         msg='if branched but not successful return `[]`, not `default`')
+        self.assertEqual(traverse_obj(_DEFAULT_DATA, ('list', Ellipsis)), [],
+                         msg='if branched but object is empty return `[]`, not `default`')
+        self.assertEqual(traverse_obj(None, Ellipsis), [],
+                         msg='if branched but object is `None` return `[]`, not `default`')
+        self.assertEqual(traverse_obj({0: None}, (0, Ellipsis)), [],
+                         msg='if branched but state is `None` return `[]`, not `default`')
+
+        branching_paths = [
+            ('fail', Ellipsis),
+            (Ellipsis, 'fail'),
+            100 * ('fail',) + (Ellipsis,),
+            (Ellipsis,) + 100 * ('fail',),
+        ]
+        for branching_path in branching_paths:
+            self.assertEqual(traverse_obj({}, branching_path), [],
+                             msg='if branched but state is `None`, return `[]` (not `default`)')
+            self.assertEqual(traverse_obj({}, 'fail', branching_path), [],
+                             msg='if branching in last alternative and previous did not match, return `[]` (not `default`)')
+            self.assertEqual(traverse_obj({0: 'x'}, 0, branching_path), 'x',
+                             msg='if branching in last alternative and previous did match, return single value')
+            self.assertEqual(traverse_obj({0: 'x'}, branching_path, 0), 'x',
+                             msg='if branching in first alternative and non-branching path does match, return single value')
+            self.assertEqual(traverse_obj({}, branching_path, 'fail'), None,
+                             msg='if branching in first alternative and non-branching path does not match, return `default`')
+
+        # Testing expected_type behavior
+        _EXPECTED_TYPE_DATA = {'str': 'str', 'int': 0}
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=str),
+                         'str', msg='accept matching `expected_type` type')
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=int),
+                         None, msg='reject non-matching `expected_type` type')
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'int', expected_type=lambda x: str(x)),
+                         '0', msg='transform type using type function')
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=lambda _: 1 / 0),
+                         None, msg='wrap expected_type function in try_call')
+        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, Ellipsis, expected_type=str),
+                         ['str'], msg='eliminate items that expected_type fails on')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2}, expected_type=int),
+                         {0: 100}, msg='type as expected_type should filter dict values')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2, 2: 'None'}, expected_type=str_or_none),
+                         {0: '100', 1: '1.2'}, msg='function as expected_type should transform dict values')
+        self.assertEqual(traverse_obj(_TEST_DATA, ({0: 1.2}, 0, set((int_or_none,))), expected_type=int),
+                         1, msg='expected_type should not filter non-final dict values')
+        self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 100, 1: 'str'}}, expected_type=int),
+                         {0: {0: 100}}, msg='expected_type should transform deep dict values')
+        self.assertEqual(traverse_obj(_TEST_DATA, [({0: '...'}, {0: '...'})], expected_type=type(Ellipsis)),
+                         [{0: Ellipsis}, {0: Ellipsis}], msg='expected_type should transform branched dict values')
+        self.assertEqual(traverse_obj({1: {3: 4}}, [(1, 2), 3], expected_type=int),
+                         [4], msg='expected_type regression for type matching in tuple branching')
+        self.assertEqual(traverse_obj(_TEST_DATA, ['data', Ellipsis], expected_type=int),
+                         [], msg='expected_type regression for type matching in dict result')
+
+        # Test get_all behavior
+        _GET_ALL_DATA = {'key': [0, 1, 2]}
+        self.assertEqual(traverse_obj(_GET_ALL_DATA, ('key', Ellipsis), get_all=False), 0,
+                         msg='if not `get_all`, return only first matching value')
+        self.assertEqual(traverse_obj(_GET_ALL_DATA, Ellipsis, get_all=False), [0, 1, 2],
+                         msg='do not overflatten if not `get_all`')
+
+        # Test casesense behavior
+        _CASESENSE_DATA = {
+            'KeY': 'value0',
+            0: {
+                'KeY': 'value1',
+                0: {'KeY': 'value2'},
+            },
+            # FULLWIDTH LATIN CAPITAL LETTER K
+            '\uff2bey': 'value3',
+        }
+        self.assertEqual(traverse_obj(_CASESENSE_DATA, 'key'), None,
+                         msg='dict keys should be case sensitive unless `casesense`')
+        self.assertEqual(traverse_obj(_CASESENSE_DATA, 'keY',
+                                      casesense=False), 'value0',
+                         msg='allow non matching key case if `casesense`')
+        self.assertEqual(traverse_obj(_CASESENSE_DATA, '\uff4bey',  # FULLWIDTH LATIN SMALL LETTER K
+                                      casesense=False), 'value3',
+                         msg='allow non matching Unicode key case if `casesense`')
+        self.assertEqual(traverse_obj(_CASESENSE_DATA, (0, ('keY',)),
+                                      casesense=False), ['value1'],
+                         msg='allow non matching key case in branch if `casesense`')
+        self.assertEqual(traverse_obj(_CASESENSE_DATA, (0, ((0, 'keY'),)),
+                                      casesense=False), ['value2'],
+                         msg='allow non matching key case in branch path if `casesense`')
+
+        # Test traverse_string behavior
+        _TRAVERSE_STRING_DATA = {'str': 'str', 1.2: 1.2}
+        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', 0)), None,
+                         msg='do not traverse into string if not `traverse_string`')
+        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', 0),
+                                      _traverse_string=True), 's',
+                         msg='traverse into string if `traverse_string`')
+        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, (1.2, 1),
+                                      _traverse_string=True), '.',
+                         msg='traverse into converted data if `traverse_string`')
+        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', Ellipsis),
+                                      _traverse_string=True), 'str',
+                         msg='`...` should result in string (same value) if `traverse_string`')
+        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', slice(0, None, 2)),
+                                      _traverse_string=True), 'sr',
+                         msg='`slice` should result in string if `traverse_string`')
+        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', lambda i, v: i or v == 's'),
+                                      _traverse_string=True), 'str',
+                         msg='function should result in string if `traverse_string`')
+        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', (0, 2)),
+                                      _traverse_string=True), ['s', 'r'],
+                         msg='branching should result in list if `traverse_string`')
+        self.assertEqual(traverse_obj({}, (0, Ellipsis), _traverse_string=True), [],
+                         msg='branching should result in list if `traverse_string`')
+        self.assertEqual(traverse_obj({}, (0, lambda x, y: True), _traverse_string=True), [],
+                         msg='branching should result in list if `traverse_string`')
+        self.assertEqual(traverse_obj({}, (0, slice(1)), _traverse_string=True), [],
+                         msg='branching should result in list if `traverse_string`')
+
+        # Test re.Match as input obj
+        mobj = re.match(r'^0(12)(?P<group>3)(4)?$', '0123')
+        self.assertEqual(traverse_obj(mobj, Ellipsis), [x for x in mobj.groups() if x is not None],
+                         msg='`...` on a `re.Match` should give its `groups()`')
+        self.assertEqual(traverse_obj(mobj, lambda k, _: k in (0, 2)), ['0123', '3'],
+                         msg='function on a `re.Match` should give groupno, value starting at 0')
+        self.assertEqual(traverse_obj(mobj, 'group'), '3',
+                         msg='str key on a `re.Match` should give group with that name')
+        self.assertEqual(traverse_obj(mobj, 2), '3',
+                         msg='int key on a `re.Match` should give group with that name')
+        self.assertEqual(traverse_obj(mobj, 'gRoUp', casesense=False), '3',
+                         msg='str key on a `re.Match` should respect casesense')
+        self.assertEqual(traverse_obj(mobj, 'fail'), None,
+                         msg='failing str key on a `re.Match` should return `default`')
+        self.assertEqual(traverse_obj(mobj, 'gRoUpS', casesense=False), None,
+                         msg='failing str key on a `re.Match` should return `default`')
+        self.assertEqual(traverse_obj(mobj, 8), None,
+                         msg='failing int key on a `re.Match` should return `default`')
+        self.assertEqual(traverse_obj(mobj, lambda k, _: k in (0, 'group')), ['0123', '3'],
+                         msg='function on a `re.Match` should give group name as well')
+
+        # Test xml.etree.ElementTree.Element as input obj
+        etree = compat_etree_fromstring('''<?xml version="1.0"?>
+        <data>
+            <country name="Liechtenstein">
+                <rank>1</rank>
+                <year>2008</year>
+                <gdppc>141100</gdppc>
+                <neighbor name="Austria" direction="E"/>
+                <neighbor name="Switzerland" direction="W"/>
+            </country>
+            <country name="Singapore">
+                <rank>4</rank>
+                <year>2011</year>
+                <gdppc>59900</gdppc>
+                <neighbor name="Malaysia" direction="N"/>
+            </country>
+            <country name="Panama">
+                <rank>68</rank>
+                <year>2011</year>
+                <gdppc>13600</gdppc>
+                <neighbor name="Costa Rica" direction="W"/>
+                <neighbor name="Colombia" direction="E"/>
+            </country>
+        </data>''')
+        self.assertEqual(traverse_obj(etree, ''), etree,
+                         msg='empty str key should return the element itself')
+        self.assertEqual(traverse_obj(etree, 'country'), list(etree),
+                         msg='str key should return all children with that tag name')
+        self.assertEqual(traverse_obj(etree, Ellipsis), list(etree),
+                         msg='`...` as key should return all children')
+        self.assertEqual(traverse_obj(etree, lambda _, x: x[0].text == '4'), [etree[1]],
+                         msg='function as key should get element as value')
+        self.assertEqual(traverse_obj(etree, lambda i, _: i == 1), [etree[1]],
+                         msg='function as key should get index as key')
+        self.assertEqual(traverse_obj(etree, 0), etree[0],
+                         msg='int key should return the nth child')
+        self.assertEqual(traverse_obj(etree, './/neighbor/@name'),
+                         ['Austria', 'Switzerland', 'Malaysia', 'Costa Rica', 'Colombia'],
+                         msg='`@<attribute>` at end of path should give that attribute')
+        self.assertEqual(traverse_obj(etree, '//neighbor/@fail'), [None, None, None, None, None],
+                         msg='`@<nonexistent>` at end of path should give `None`')
+        self.assertEqual(traverse_obj(etree, ('//neighbor/@', 2)), {'name': 'Malaysia', 'direction': 'N'},
+                         msg='`@` should give the full attribute dict')
+        self.assertEqual(traverse_obj(etree, '//year/text()'), ['2008', '2011', '2011'],
+                         msg='`text()` at end of path should give the inner text')
+        self.assertEqual(traverse_obj(etree, '//*[@direction]/@direction'), ['E', 'W', 'N', 'W', 'E'],
+                         msg='full python xpath features should be supported')
+        self.assertEqual(traverse_obj(etree, (0, '@name')), 'Liechtenstein',
+                         msg='special transformations should act on current element')
+        self.assertEqual(traverse_obj(etree, ('country', 0, Ellipsis, 'text()', T(int_or_none))), [1, 2008, 141100],
+                         msg='special transformations should act on current element')
+
+    def test_traversal_unbranching(self):
+        self.assertEqual(traverse_obj(_TEST_DATA, [(100, 1.2), all]), [100, 1.2],
+                         msg='`all` should give all results as list')
+        self.assertEqual(traverse_obj(_TEST_DATA, [(100, 1.2), any]), 100,
+                         msg='`any` should give the first result')
+        self.assertEqual(traverse_obj(_TEST_DATA, [100, all]), [100],
+                         msg='`all` should give list if non branching')
+        self.assertEqual(traverse_obj(_TEST_DATA, [100, any]), 100,
+                         msg='`any` should give single item if non branching')
+        self.assertEqual(traverse_obj(_TEST_DATA, [('dict', 'None', 100), all]), [100],
+                         msg='`all` should filter `None` and empty dict')
+        self.assertEqual(traverse_obj(_TEST_DATA, [('dict', 'None', 100), any]), 100,
+                         msg='`any` should filter `None` and empty dict')
+        self.assertEqual(traverse_obj(_TEST_DATA, [{
+            'all': [('dict', 'None', 100, 1.2), all],
+            'any': [('dict', 'None', 100, 1.2), any],
+        }]), {'all': [100, 1.2], 'any': 100},
+            msg='`all`/`any` should apply to each dict path separately')
+        self.assertEqual(traverse_obj(_TEST_DATA, [{
+            'all': [('dict', 'None', 100, 1.2), all],
+            'any': [('dict', 'None', 100, 1.2), any],
+        }], get_all=False), {'all': [100, 1.2], 'any': 100},
+            msg='`all`/`any` should apply to dict regardless of `get_all`')
+        self.assertIs(traverse_obj(_TEST_DATA, [('dict', 'None', 100, 1.2), all, T(float)]), None,
+                      msg='`all` should reset branching status')
+        self.assertIs(traverse_obj(_TEST_DATA, [('dict', 'None', 100, 1.2), any, T(float)]), None,
+                      msg='`any` should reset branching status')
+        self.assertEqual(traverse_obj(_TEST_DATA, [('dict', 'None', 100, 1.2), all, Ellipsis, T(float)]), [1.2],
+                         msg='`all` should allow further branching')
+        self.assertEqual(traverse_obj(_TEST_DATA, [('dict', 'None', 'urls', 'data'), any, Ellipsis, 'index']), [0, 1],
+                         msg='`any` should allow further branching')
+
+    def test_traversal_morsel(self):
+        values = {
+            'expires': 'a',
+            'path': 'b',
+            'comment': 'c',
+            'domain': 'd',
+            'max-age': 'e',
+            'secure': 'f',
+            'httponly': 'g',
+            'version': 'h',
+            'samesite': 'i',
+        }
+        # SameSite added in Py3.8, breaks .update for 3.5-3.7
+        if sys.version_info < (3, 8):
+            del values['samesite']
+        morsel = compat_http_cookies.Morsel()
+        morsel.set(str('item_key'), 'item_value', 'coded_value')
+        morsel.update(values)
+        values['key'] = str('item_key')
+        values['value'] = 'item_value'
+        values = dict((str(k), v) for k, v in values.items())
+        # make test pass even without ordered dict
+        value_set = set(values.values())
+
+        for key, value in values.items():
+            self.assertEqual(traverse_obj(morsel, key), value,
+                             msg='Morsel should provide access to all values')
+        self.assertEqual(set(traverse_obj(morsel, Ellipsis)), value_set,
+                         msg='`...` should yield all values')
+        self.assertEqual(set(traverse_obj(morsel, lambda k, v: True)), value_set,
+                         msg='function key should yield all values')
+        self.assertIs(traverse_obj(morsel, [(None,), any]), morsel,
+                      msg='Morsel should not be implicitly changed to dict on usage')
+
+    def test_get_first(self):
+        self.assertEqual(get_first([{'a': None}, {'a': 'spam'}], 'a'), 'spam')
+
+    def test_dict_get(self):
+        FALSE_VALUES = {
+            'none': None,
+            'false': False,
+            'zero': 0,
+            'empty_string': '',
+            'empty_list': [],
+        }
+        d = FALSE_VALUES.copy()
+        d['a'] = 42
+        self.assertEqual(dict_get(d, 'a'), 42)
+        self.assertEqual(dict_get(d, 'b'), None)
+        self.assertEqual(dict_get(d, 'b', 42), 42)
+        self.assertEqual(dict_get(d, ('a', )), 42)
+        self.assertEqual(dict_get(d, ('b', 'a', )), 42)
+        self.assertEqual(dict_get(d, ('b', 'c', 'a', 'd', )), 42)
+        self.assertEqual(dict_get(d, ('b', 'c', )), None)
+        self.assertEqual(dict_get(d, ('b', 'c', ), 42), 42)
+        for key, false_value in FALSE_VALUES.items():
+            self.assertEqual(dict_get(d, ('b', 'c', key, )), None)
+            self.assertEqual(dict_get(d, ('b', 'c', key, ), skip_false_values=False), false_value)
+
+
+if __name__ == '__main__':
+    unittest.main()
diff --git a/test/test_utils.py b/test/test_utils.py
index 179d21cf5..de7fe80b8 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -14,7 +14,6 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 import io
 import itertools
 import json
-import re
 import xml.etree.ElementTree
 
 from youtube_dl.utils import (
@@ -28,7 +27,6 @@ from youtube_dl.utils import (
     DateRange,
     detect_exe_version,
     determine_ext,
-    dict_get,
     encode_base_n,
     encode_compat_str,
     encodeFilename,
@@ -44,7 +42,6 @@ from youtube_dl.utils import (
     get_element_by_attribute,
     get_elements_by_class,
     get_elements_by_attribute,
-    get_first,
     InAdvancePagedList,
     int_or_none,
     intlist_to_bytes,
@@ -84,14 +81,11 @@ from youtube_dl.utils import (
     sanitized_Request,
     shell_quote,
     smuggle_url,
-    str_or_none,
     str_to_int,
     strip_jsonp,
     strip_or_none,
     subtitles_filename,
-    T,
     timeconvert,
-    traverse_obj,
     try_call,
     unescapeHTML,
     unified_strdate,
@@ -123,7 +117,6 @@ from youtube_dl.compat import (
     compat_chr,
     compat_etree_fromstring,
     compat_getenv,
-    compat_http_cookies,
     compat_os_name,
     compat_setenv,
     compat_str,
@@ -1685,481 +1678,5 @@ Line 1
             from_dict={'a': 'c', 'c': [], 'b': 'd', 'd': None}), 'c-d')
 
 
-class TestTraversal(unittest.TestCase):
-    str = compat_str
-    _TEST_DATA = {
-        100: 100,
-        1.2: 1.2,
-        'str': 'str',
-        'None': None,
-        '...': Ellipsis,
-        'urls': [
-            {'index': 0, 'url': 'https://www.example.com/0'},
-            {'index': 1, 'url': 'https://www.example.com/1'},
-        ],
-        'data': (
-            {'index': 2},
-            {'index': 3},
-        ),
-        'dict': {},
-    }
-
-    # yt-dlp shim
-    def assertCountEqual(self, expected, got, msg='count should be the same'):
-        return self.assertEqual(len(tuple(expected)), len(tuple(got)), msg=msg)
-
-    def assertMaybeCountEqual(self, *args, **kwargs):
-        if sys.version_info < (3, 7):
-            # random dict order
-            return self.assertCountEqual(*args, **kwargs)
-        else:
-            return self.assertEqual(*args, **kwargs)
-
-    def test_traverse_obj(self):
-        str = self.str
-        _TEST_DATA = self._TEST_DATA
-
-        # define a pukka Iterable
-        def iter_range(stop):
-            for from_ in range(stop):
-                yield from_
-
-        # Test base functionality
-        self.assertEqual(traverse_obj(_TEST_DATA, ('str',)), 'str',
-                         msg='allow tuple path')
-        self.assertEqual(traverse_obj(_TEST_DATA, ['str']), 'str',
-                         msg='allow list path')
-        self.assertEqual(traverse_obj(_TEST_DATA, (value for value in ("str",))), 'str',
-                         msg='allow iterable path')
-        self.assertEqual(traverse_obj(_TEST_DATA, 'str'), 'str',
-                         msg='single items should be treated as a path')
-        self.assertEqual(traverse_obj(_TEST_DATA, None), _TEST_DATA)
-        self.assertEqual(traverse_obj(_TEST_DATA, 100), 100)
-        self.assertEqual(traverse_obj(_TEST_DATA, 1.2), 1.2)
-
-        # Test Ellipsis behavior
-        self.assertCountEqual(traverse_obj(_TEST_DATA, Ellipsis),
-                              (item for item in _TEST_DATA.values() if item not in (None, {})),
-                              msg='`...` should give all non-discarded values')
-        self.assertCountEqual(traverse_obj(_TEST_DATA, ('urls', 0, Ellipsis)), _TEST_DATA['urls'][0].values(),
-                              msg='`...` selection for dicts should select all values')
-        self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, Ellipsis, 'url')),
-                         ['https://www.example.com/0', 'https://www.example.com/1'],
-                         msg='nested `...` queries should work')
-        self.assertCountEqual(traverse_obj(_TEST_DATA, (Ellipsis, Ellipsis, 'index')), iter_range(4),
-                              msg='`...` query result should be flattened')
-        self.assertEqual(traverse_obj(iter(range(4)), Ellipsis), list(range(4)),
-                         msg='`...` should accept iterables')
-
-        # Test function as key
-        self.assertEqual(traverse_obj(_TEST_DATA, lambda x, y: x == 'urls' and isinstance(y, list)),
-                         [_TEST_DATA['urls']],
-                         msg='function as query key should perform a filter based on (key, value)')
-        self.assertCountEqual(traverse_obj(_TEST_DATA, lambda _, x: isinstance(x[0], str)), set(('str',)),
-                              msg='exceptions in the query function should be caught')
-        self.assertEqual(traverse_obj(iter(range(4)), lambda _, x: x % 2 == 0), [0, 2],
-                         msg='function key should accept iterables')
-        if __debug__:
-            with self.assertRaises(Exception, msg='Wrong function signature should raise in debug'):
-                traverse_obj(_TEST_DATA, lambda a: Ellipsis)
-            with self.assertRaises(Exception, msg='Wrong function signature should raise in debug'):
-                traverse_obj(_TEST_DATA, lambda a, b, c: Ellipsis)
-
-        # Test set as key (transformation/type, like `expected_type`)
-        self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str.upper), )), ['STR'],
-                         msg='Function in set should be a transformation')
-        self.assertEqual(traverse_obj(_TEST_DATA, ('fail', T(lambda _: 'const'))), 'const',
-                         msg='Function in set should always be called')
-        self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str))), ['str'],
-                         msg='Type in set should be a type filter')
-        self.assertMaybeCountEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str, int))), [100, 'str'],
-                                   msg='Multiple types in set should be a type filter')
-        self.assertEqual(traverse_obj(_TEST_DATA, T(dict)), _TEST_DATA,
-                         msg='A single set should be wrapped into a path')
-        self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str.upper))), ['STR'],
-                         msg='Transformation function should not raise')
-        self.assertMaybeCountEqual(traverse_obj(_TEST_DATA, (Ellipsis, T(str_or_none))),
-                                   [item for item in map(str_or_none, _TEST_DATA.values()) if item is not None],
-                                   msg='Function in set should be a transformation')
-        if __debug__:
-            with self.assertRaises(Exception, msg='Sets with length != 1 should raise in debug'):
-                traverse_obj(_TEST_DATA, set())
-            with self.assertRaises(Exception, msg='Sets with length != 1 should raise in debug'):
-                traverse_obj(_TEST_DATA, set((str.upper, str)))
-
-        # Test `slice` as a key
-        _SLICE_DATA = [0, 1, 2, 3, 4]
-        self.assertEqual(traverse_obj(_TEST_DATA, ('dict', slice(1))), None,
-                         msg='slice on a dictionary should not throw')
-        self.assertEqual(traverse_obj(_SLICE_DATA, slice(1)), _SLICE_DATA[:1],
-                         msg='slice key should apply slice to sequence')
-        self.assertEqual(traverse_obj(_SLICE_DATA, slice(1, 2)), _SLICE_DATA[1:2],
-                         msg='slice key should apply slice to sequence')
-        self.assertEqual(traverse_obj(_SLICE_DATA, slice(1, 4, 2)), _SLICE_DATA[1:4:2],
-                         msg='slice key should apply slice to sequence')
-
-        # Test alternative paths
-        self.assertEqual(traverse_obj(_TEST_DATA, 'fail', 'str'), 'str',
-                         msg='multiple `paths` should be treated as alternative paths')
-        self.assertEqual(traverse_obj(_TEST_DATA, 'str', 100), 'str',
-                         msg='alternatives should exit early')
-        self.assertEqual(traverse_obj(_TEST_DATA, 'fail', 'fail'), None,
-                         msg='alternatives should return `default` if exhausted')
-        self.assertEqual(traverse_obj(_TEST_DATA, (Ellipsis, 'fail'), 100), 100,
-                         msg='alternatives should track their own branching return')
-        self.assertEqual(traverse_obj(_TEST_DATA, ('dict', Ellipsis), ('data', Ellipsis)), list(_TEST_DATA['data']),
-                         msg='alternatives on empty objects should search further')
-
-        # Test branch and path nesting
-        self.assertEqual(traverse_obj(_TEST_DATA, ('urls', (3, 0), 'url')), ['https://www.example.com/0'],
-                         msg='tuple as key should be treated as branches')
-        self.assertEqual(traverse_obj(_TEST_DATA, ('urls', [3, 0], 'url')), ['https://www.example.com/0'],
-                         msg='list as key should be treated as branches')
-        self.assertEqual(traverse_obj(_TEST_DATA, ('urls', ((1, 'fail'), (0, 'url')))), ['https://www.example.com/0'],
-                         msg='double nesting in path should be treated as paths')
-        self.assertEqual(traverse_obj(['0', [1, 2]], [(0, 1), 0]), [1],
-                         msg='do not fail early on branching')
-        self.assertCountEqual(traverse_obj(_TEST_DATA, ('urls', ((1, ('fail', 'url')), (0, 'url')))),
-                              ['https://www.example.com/0', 'https://www.example.com/1'],
-                              msg='triple nesting in path should be treated as branches')
-        self.assertEqual(traverse_obj(_TEST_DATA, ('urls', ('fail', (Ellipsis, 'url')))),
-                         ['https://www.example.com/0', 'https://www.example.com/1'],
-                         msg='ellipsis as branch path start gets flattened')
-
-        # Test dictionary as key
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2}), {0: 100, 1: 1.2},
-                         msg='dict key should result in a dict with the same keys')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', 0, 'url')}),
-                         {0: 'https://www.example.com/0'},
-                         msg='dict key should allow paths')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', (3, 0), 'url')}),
-                         {0: ['https://www.example.com/0']},
-                         msg='tuple in dict path should be treated as branches')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', ((1, 'fail'), (0, 'url')))}),
-                         {0: ['https://www.example.com/0']},
-                         msg='double nesting in dict path should be treated as paths')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('urls', ((1, ('fail', 'url')), (0, 'url')))}),
-                         {0: ['https://www.example.com/1', 'https://www.example.com/0']},
-                         msg='triple nesting in dict path should be treated as branches')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'fail'}), {},
-                         msg='remove `None` values when top level dict key fails')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'fail'}, default=Ellipsis), {0: Ellipsis},
-                         msg='use `default` if key fails and `default`')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}), {},
-                         msg='remove empty values when dict key')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: 'dict'}, default=Ellipsis), {0: Ellipsis},
-                         msg='use `default` when dict key and a default')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 'fail'}}), {},
-                         msg='remove empty values when nested dict key fails')
-        self.assertEqual(traverse_obj(None, {0: 'fail'}), {},
-                         msg='default to dict if pruned')
-        self.assertEqual(traverse_obj(None, {0: 'fail'}, default=Ellipsis), {0: Ellipsis},
-                         msg='default to dict if pruned and default is given')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 'fail'}}, default=Ellipsis), {0: {0: Ellipsis}},
-                         msg='use nested `default` when nested dict key fails and `default`')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: ('dict', Ellipsis)}), {},
-                         msg='remove key if branch in dict key not successful')
-
-        # Testing default parameter behavior
-        _DEFAULT_DATA = {'None': None, 'int': 0, 'list': []}
-        self.assertEqual(traverse_obj(_DEFAULT_DATA, 'fail'), None,
-                         msg='default value should be `None`')
-        self.assertEqual(traverse_obj(_DEFAULT_DATA, 'fail', 'fail', default=Ellipsis), Ellipsis,
-                         msg='chained fails should result in default')
-        self.assertEqual(traverse_obj(_DEFAULT_DATA, 'None', 'int'), 0,
-                         msg='should not short cirquit on `None`')
-        self.assertEqual(traverse_obj(_DEFAULT_DATA, 'fail', default=1), 1,
-                         msg='invalid dict key should result in `default`')
-        self.assertEqual(traverse_obj(_DEFAULT_DATA, 'None', default=1), 1,
-                         msg='`None` is a deliberate sentinel and should become `default`')
-        self.assertEqual(traverse_obj(_DEFAULT_DATA, ('list', 10)), None,
-                         msg='`IndexError` should result in `default`')
-        self.assertEqual(traverse_obj(_DEFAULT_DATA, (Ellipsis, 'fail'), default=1), 1,
-                         msg='if branched but not successful return `default` if defined, not `[]`')
-        self.assertEqual(traverse_obj(_DEFAULT_DATA, (Ellipsis, 'fail'), default=None), None,
-                         msg='if branched but not successful return `default` even if `default` is `None`')
-        self.assertEqual(traverse_obj(_DEFAULT_DATA, (Ellipsis, 'fail')), [],
-                         msg='if branched but not successful return `[]`, not `default`')
-        self.assertEqual(traverse_obj(_DEFAULT_DATA, ('list', Ellipsis)), [],
-                         msg='if branched but object is empty return `[]`, not `default`')
-        self.assertEqual(traverse_obj(None, Ellipsis), [],
-                         msg='if branched but object is `None` return `[]`, not `default`')
-        self.assertEqual(traverse_obj({0: None}, (0, Ellipsis)), [],
-                         msg='if branched but state is `None` return `[]`, not `default`')
-
-        branching_paths = [
-            ('fail', Ellipsis),
-            (Ellipsis, 'fail'),
-            100 * ('fail',) + (Ellipsis,),
-            (Ellipsis,) + 100 * ('fail',),
-        ]
-        for branching_path in branching_paths:
-            self.assertEqual(traverse_obj({}, branching_path), [],
-                             msg='if branched but state is `None`, return `[]` (not `default`)')
-            self.assertEqual(traverse_obj({}, 'fail', branching_path), [],
-                             msg='if branching in last alternative and previous did not match, return `[]` (not `default`)')
-            self.assertEqual(traverse_obj({0: 'x'}, 0, branching_path), 'x',
-                             msg='if branching in last alternative and previous did match, return single value')
-            self.assertEqual(traverse_obj({0: 'x'}, branching_path, 0), 'x',
-                             msg='if branching in first alternative and non-branching path does match, return single value')
-            self.assertEqual(traverse_obj({}, branching_path, 'fail'), None,
-                             msg='if branching in first alternative and non-branching path does not match, return `default`')
-
-        # Testing expected_type behavior
-        _EXPECTED_TYPE_DATA = {'str': 'str', 'int': 0}
-        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=str),
-                         'str', msg='accept matching `expected_type` type')
-        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=int),
-                         None, msg='reject non-matching `expected_type` type')
-        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'int', expected_type=lambda x: str(x)),
-                         '0', msg='transform type using type function')
-        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, 'str', expected_type=lambda _: 1 / 0),
-                         None, msg='wrap expected_type function in try_call')
-        self.assertEqual(traverse_obj(_EXPECTED_TYPE_DATA, Ellipsis, expected_type=str),
-                         ['str'], msg='eliminate items that expected_type fails on')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2}, expected_type=int),
-                         {0: 100}, msg='type as expected_type should filter dict values')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: 100, 1: 1.2, 2: 'None'}, expected_type=str_or_none),
-                         {0: '100', 1: '1.2'}, msg='function as expected_type should transform dict values')
-        self.assertEqual(traverse_obj(_TEST_DATA, ({0: 1.2}, 0, set((int_or_none,))), expected_type=int),
-                         1, msg='expected_type should not filter non-final dict values')
-        self.assertEqual(traverse_obj(_TEST_DATA, {0: {0: 100, 1: 'str'}}, expected_type=int),
-                         {0: {0: 100}}, msg='expected_type should transform deep dict values')
-        self.assertEqual(traverse_obj(_TEST_DATA, [({0: '...'}, {0: '...'})], expected_type=type(Ellipsis)),
-                         [{0: Ellipsis}, {0: Ellipsis}], msg='expected_type should transform branched dict values')
-        self.assertEqual(traverse_obj({1: {3: 4}}, [(1, 2), 3], expected_type=int),
-                         [4], msg='expected_type regression for type matching in tuple branching')
-        self.assertEqual(traverse_obj(_TEST_DATA, ['data', Ellipsis], expected_type=int),
-                         [], msg='expected_type regression for type matching in dict result')
-
-        # Test get_all behavior
-        _GET_ALL_DATA = {'key': [0, 1, 2]}
-        self.assertEqual(traverse_obj(_GET_ALL_DATA, ('key', Ellipsis), get_all=False), 0,
-                         msg='if not `get_all`, return only first matching value')
-        self.assertEqual(traverse_obj(_GET_ALL_DATA, Ellipsis, get_all=False), [0, 1, 2],
-                         msg='do not overflatten if not `get_all`')
-
-        # Test casesense behavior
-        _CASESENSE_DATA = {
-            'KeY': 'value0',
-            0: {
-                'KeY': 'value1',
-                0: {'KeY': 'value2'},
-            },
-            # FULLWIDTH LATIN CAPITAL LETTER K
-            '\uff2bey': 'value3',
-        }
-        self.assertEqual(traverse_obj(_CASESENSE_DATA, 'key'), None,
-                         msg='dict keys should be case sensitive unless `casesense`')
-        self.assertEqual(traverse_obj(_CASESENSE_DATA, 'keY',
-                                      casesense=False), 'value0',
-                         msg='allow non matching key case if `casesense`')
-        self.assertEqual(traverse_obj(_CASESENSE_DATA, '\uff4bey',  # FULLWIDTH LATIN SMALL LETTER K
-                                      casesense=False), 'value3',
-                         msg='allow non matching Unicode key case if `casesense`')
-        self.assertEqual(traverse_obj(_CASESENSE_DATA, (0, ('keY',)),
-                                      casesense=False), ['value1'],
-                         msg='allow non matching key case in branch if `casesense`')
-        self.assertEqual(traverse_obj(_CASESENSE_DATA, (0, ((0, 'keY'),)),
-                                      casesense=False), ['value2'],
-                         msg='allow non matching key case in branch path if `casesense`')
-
-        # Test traverse_string behavior
-        _TRAVERSE_STRING_DATA = {'str': 'str', 1.2: 1.2}
-        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', 0)), None,
-                         msg='do not traverse into string if not `traverse_string`')
-        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', 0),
-                                      _traverse_string=True), 's',
-                         msg='traverse into string if `traverse_string`')
-        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, (1.2, 1),
-                                      _traverse_string=True), '.',
-                         msg='traverse into converted data if `traverse_string`')
-        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', Ellipsis),
-                                      _traverse_string=True), 'str',
-                         msg='`...` should result in string (same value) if `traverse_string`')
-        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', slice(0, None, 2)),
-                                      _traverse_string=True), 'sr',
-                         msg='`slice` should result in string if `traverse_string`')
-        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', lambda i, v: i or v == 's'),
-                                      _traverse_string=True), 'str',
-                         msg='function should result in string if `traverse_string`')
-        self.assertEqual(traverse_obj(_TRAVERSE_STRING_DATA, ('str', (0, 2)),
-                                      _traverse_string=True), ['s', 'r'],
-                         msg='branching should result in list if `traverse_string`')
-        self.assertEqual(traverse_obj({}, (0, Ellipsis), _traverse_string=True), [],
-                         msg='branching should result in list if `traverse_string`')
-        self.assertEqual(traverse_obj({}, (0, lambda x, y: True), _traverse_string=True), [],
-                         msg='branching should result in list if `traverse_string`')
-        self.assertEqual(traverse_obj({}, (0, slice(1)), _traverse_string=True), [],
-                         msg='branching should result in list if `traverse_string`')
-
-        # Test re.Match as input obj
-        mobj = re.match(r'^0(12)(?P<group>3)(4)?$', '0123')
-        self.assertEqual(traverse_obj(mobj, Ellipsis), [x for x in mobj.groups() if x is not None],
-                         msg='`...` on a `re.Match` should give its `groups()`')
-        self.assertEqual(traverse_obj(mobj, lambda k, _: k in (0, 2)), ['0123', '3'],
-                         msg='function on a `re.Match` should give groupno, value starting at 0')
-        self.assertEqual(traverse_obj(mobj, 'group'), '3',
-                         msg='str key on a `re.Match` should give group with that name')
-        self.assertEqual(traverse_obj(mobj, 2), '3',
-                         msg='int key on a `re.Match` should give group with that name')
-        self.assertEqual(traverse_obj(mobj, 'gRoUp', casesense=False), '3',
-                         msg='str key on a `re.Match` should respect casesense')
-        self.assertEqual(traverse_obj(mobj, 'fail'), None,
-                         msg='failing str key on a `re.Match` should return `default`')
-        self.assertEqual(traverse_obj(mobj, 'gRoUpS', casesense=False), None,
-                         msg='failing str key on a `re.Match` should return `default`')
-        self.assertEqual(traverse_obj(mobj, 8), None,
-                         msg='failing int key on a `re.Match` should return `default`')
-        self.assertEqual(traverse_obj(mobj, lambda k, _: k in (0, 'group')), ['0123', '3'],
-                         msg='function on a `re.Match` should give group name as well')
-
-        # Test xml.etree.ElementTree.Element as input obj
-        etree = compat_etree_fromstring('''<?xml version="1.0"?>
-        <data>
-            <country name="Liechtenstein">
-                <rank>1</rank>
-                <year>2008</year>
-                <gdppc>141100</gdppc>
-                <neighbor name="Austria" direction="E"/>
-                <neighbor name="Switzerland" direction="W"/>
-            </country>
-            <country name="Singapore">
-                <rank>4</rank>
-                <year>2011</year>
-                <gdppc>59900</gdppc>
-                <neighbor name="Malaysia" direction="N"/>
-            </country>
-            <country name="Panama">
-                <rank>68</rank>
-                <year>2011</year>
-                <gdppc>13600</gdppc>
-                <neighbor name="Costa Rica" direction="W"/>
-                <neighbor name="Colombia" direction="E"/>
-            </country>
-        </data>''')
-        self.assertEqual(traverse_obj(etree, ''), etree,
-                         msg='empty str key should return the element itself')
-        self.assertEqual(traverse_obj(etree, 'country'), list(etree),
-                         msg='str key should return all children with that tag name')
-        self.assertEqual(traverse_obj(etree, Ellipsis), list(etree),
-                         msg='`...` as key should return all children')
-        self.assertEqual(traverse_obj(etree, lambda _, x: x[0].text == '4'), [etree[1]],
-                         msg='function as key should get element as value')
-        self.assertEqual(traverse_obj(etree, lambda i, _: i == 1), [etree[1]],
-                         msg='function as key should get index as key')
-        self.assertEqual(traverse_obj(etree, 0), etree[0],
-                         msg='int key should return the nth child')
-        self.assertEqual(traverse_obj(etree, './/neighbor/@name'),
-                         ['Austria', 'Switzerland', 'Malaysia', 'Costa Rica', 'Colombia'],
-                         msg='`@<attribute>` at end of path should give that attribute')
-        self.assertEqual(traverse_obj(etree, '//neighbor/@fail'), [None, None, None, None, None],
-                         msg='`@<nonexistent>` at end of path should give `None`')
-        self.assertEqual(traverse_obj(etree, ('//neighbor/@', 2)), {'name': 'Malaysia', 'direction': 'N'},
-                         msg='`@` should give the full attribute dict')
-        self.assertEqual(traverse_obj(etree, '//year/text()'), ['2008', '2011', '2011'],
-                         msg='`text()` at end of path should give the inner text')
-        self.assertEqual(traverse_obj(etree, '//*[@direction]/@direction'), ['E', 'W', 'N', 'W', 'E'],
-                         msg='full python xpath features should be supported')
-        self.assertEqual(traverse_obj(etree, (0, '@name')), 'Liechtenstein',
-                         msg='special transformations should act on current element')
-        self.assertEqual(traverse_obj(etree, ('country', 0, Ellipsis, 'text()', T(int_or_none))), [1, 2008, 141100],
-                         msg='special transformations should act on current element')
-
-    def test_traversal_unbranching(self):
-        # str = self.str
-        _TEST_DATA = self._TEST_DATA
-
-        self.assertEqual(traverse_obj(_TEST_DATA, [(100, 1.2), all]), [100, 1.2],
-                         msg='`all` should give all results as list')
-        self.assertEqual(traverse_obj(_TEST_DATA, [(100, 1.2), any]), 100,
-                         msg='`any` should give the first result')
-        self.assertEqual(traverse_obj(_TEST_DATA, [100, all]), [100],
-                         msg='`all` should give list if non branching')
-        self.assertEqual(traverse_obj(_TEST_DATA, [100, any]), 100,
-                         msg='`any` should give single item if non branching')
-        self.assertEqual(traverse_obj(_TEST_DATA, [('dict', 'None', 100), all]), [100],
-                         msg='`all` should filter `None` and empty dict')
-        self.assertEqual(traverse_obj(_TEST_DATA, [('dict', 'None', 100), any]), 100,
-                         msg='`any` should filter `None` and empty dict')
-        self.assertEqual(traverse_obj(_TEST_DATA, [{
-            'all': [('dict', 'None', 100, 1.2), all],
-            'any': [('dict', 'None', 100, 1.2), any],
-        }]), {'all': [100, 1.2], 'any': 100},
-            msg='`all`/`any` should apply to each dict path separately')
-        self.assertEqual(traverse_obj(_TEST_DATA, [{
-            'all': [('dict', 'None', 100, 1.2), all],
-            'any': [('dict', 'None', 100, 1.2), any],
-        }], get_all=False), {'all': [100, 1.2], 'any': 100},
-            msg='`all`/`any` should apply to dict regardless of `get_all`')
-        self.assertIs(traverse_obj(_TEST_DATA, [('dict', 'None', 100, 1.2), all, T(float)]), None,
-                      msg='`all` should reset branching status')
-        self.assertIs(traverse_obj(_TEST_DATA, [('dict', 'None', 100, 1.2), any, T(float)]), None,
-                      msg='`any` should reset branching status')
-        self.assertEqual(traverse_obj(_TEST_DATA, [('dict', 'None', 100, 1.2), all, Ellipsis, T(float)]), [1.2],
-                         msg='`all` should allow further branching')
-        self.assertEqual(traverse_obj(_TEST_DATA, [('dict', 'None', 'urls', 'data'), any, Ellipsis, 'index']), [0, 1],
-                         msg='`any` should allow further branching')
-
-    def test_traversal_morsel(self):
-        values = {
-            'expires': 'a',
-            'path': 'b',
-            'comment': 'c',
-            'domain': 'd',
-            'max-age': 'e',
-            'secure': 'f',
-            'httponly': 'g',
-            'version': 'h',
-            'samesite': 'i',
-        }
-        # SameSite added in Py3.8, breaks .update for 3.5-3.7
-        if sys.version_info < (3, 8):
-            del values['samesite']
-        morsel = compat_http_cookies.Morsel()
-        morsel.set(str('item_key'), 'item_value', 'coded_value')
-        morsel.update(values)
-        values['key'] = str('item_key')
-        values['value'] = 'item_value'
-        values = dict((str(k), v) for k, v in values.items())
-        # make test pass even without ordered dict
-        value_set = set(values.values())
-
-        for key, value in values.items():
-            self.assertEqual(traverse_obj(morsel, key), value,
-                             msg='Morsel should provide access to all values')
-        self.assertEqual(set(traverse_obj(morsel, Ellipsis)), value_set,
-                         msg='`...` should yield all values')
-        self.assertEqual(set(traverse_obj(morsel, lambda k, v: True)), value_set,
-                         msg='function key should yield all values')
-        self.assertIs(traverse_obj(morsel, [(None,), any]), morsel,
-                      msg='Morsel should not be implicitly changed to dict on usage')
-
-    def test_get_first(self):
-        self.assertEqual(get_first([{'a': None}, {'a': 'spam'}], 'a'), 'spam')
-
-    def test_dict_get(self):
-        FALSE_VALUES = {
-            'none': None,
-            'false': False,
-            'zero': 0,
-            'empty_string': '',
-            'empty_list': [],
-        }
-        d = FALSE_VALUES.copy()
-        d['a'] = 42
-        self.assertEqual(dict_get(d, 'a'), 42)
-        self.assertEqual(dict_get(d, 'b'), None)
-        self.assertEqual(dict_get(d, 'b', 42), 42)
-        self.assertEqual(dict_get(d, ('a', )), 42)
-        self.assertEqual(dict_get(d, ('b', 'a', )), 42)
-        self.assertEqual(dict_get(d, ('b', 'c', 'a', 'd', )), 42)
-        self.assertEqual(dict_get(d, ('b', 'c', )), None)
-        self.assertEqual(dict_get(d, ('b', 'c', ), 42), 42)
-        for key, false_value in FALSE_VALUES.items():
-            self.assertEqual(dict_get(d, ('b', 'c', key, )), None)
-            self.assertEqual(dict_get(d, ('b', 'c', key, ), skip_false_values=False), false_value)
-
-
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/traversal.py b/youtube_dl/traversal.py
new file mode 100644
index 000000000..834cfef7f
--- /dev/null
+++ b/youtube_dl/traversal.py
@@ -0,0 +1,10 @@
+# coding: utf-8
+
+# TODO: move these utils.fns here and move import to utils
+# flake8: noqa
+from .utils import (
+    dict_get,
+    get_first,
+    T,
+    traverse_obj,
+)

From 768ccccd9b18bc48d129b12d14eace4ebb3655d8 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 28 May 2024 15:59:34 +0100
Subject: [PATCH 756/860] [compat] Avoid type comparison in `compat_ord` NB
 This isn't actually a compat fn; it should be utils.int_from_int_or_char

---
 youtube_dl/compat.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 0371896ab..ed1a33cf2 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2970,7 +2970,7 @@ except (AssertionError, UnicodeEncodeError):
 
 
 def compat_ord(c):
-    if type(c) is int:
+    if isinstance(c, int):
         return c
     else:
         return ord(c)

From 21924742f79ccbd62d16ef4120518c6a5da8614e Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 26 Apr 2024 18:57:44 +0100
Subject: [PATCH 757/860] [InfoExtractor] Misc yt-dlp back-ports, etc * add
 _yes_playlist() method * avoid crash using _NETRC_MACHINE * use
 _search_json() in _search_nextjs_data() * _search_nextjs_data() default is
 JSON, not text * test for above

---
 test/test_InfoExtractor.py     |  3 ++
 youtube_dl/extractor/common.py | 63 +++++++++++++++++++++++++---------
 2 files changed, 50 insertions(+), 16 deletions(-)

diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py
index d55d6ad54..09100a1d6 100644
--- a/test/test_InfoExtractor.py
+++ b/test/test_InfoExtractor.py
@@ -153,6 +153,9 @@ class TestInfoExtractor(unittest.TestCase):
 '''
         search = self.ie._search_nextjs_data(html, 'testID')
         self.assertEqual(search['props']['pageProps']['video']['id'], 'testid')
+        search = self.ie._search_nextjs_data(
+            'no next.js data here, move along', 'testID', default={'status': 0})
+        self.assertEqual(search['status'], 0)
 
     def test_search_nuxt_data(self):
         html = '''
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 7fae9e57b..b10e84416 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -1169,10 +1169,10 @@ class InfoExtractor(object):
     def _get_netrc_login_info(self, netrc_machine=None):
         username = None
         password = None
-        netrc_machine = netrc_machine or self._NETRC_MACHINE
 
         if self._downloader.params.get('usenetrc', False):
             try:
+                netrc_machine = netrc_machine or self._NETRC_MACHINE
                 info = netrc.netrc().authenticators(netrc_machine)
                 if info is not None:
                     username = info[0]
@@ -1180,7 +1180,7 @@ class InfoExtractor(object):
                 else:
                     raise netrc.NetrcParseError(
                         'No authenticators for %s' % netrc_machine)
-            except (IOError, netrc.NetrcParseError) as err:
+            except (AttributeError, IOError, netrc.NetrcParseError) as err:
                 self._downloader.report_warning(
                     'parsing .netrc: %s' % error_to_compat_str(err))
 
@@ -1490,14 +1490,18 @@ class InfoExtractor(object):
         return dict((k, v) for k, v in info.items() if v is not None)
 
     def _search_nextjs_data(self, webpage, video_id, **kw):
-        nkw = dict((k, v) for k, v in kw.items() if k in ('transform_source', 'fatal'))
-        kw.pop('transform_source', None)
-        next_data = self._search_regex(
-            r'''<script[^>]+\bid\s*=\s*('|")__NEXT_DATA__\1[^>]*>(?P<nd>[^<]+)</script>''',
-            webpage, 'next.js data', group='nd', **kw)
-        if not next_data:
-            return {}
-        return self._parse_json(next_data, video_id, **nkw)
+        # ..., *, transform_source=None, fatal=True, default=NO_DEFAULT
+
+        # TODO: remove this backward compat
+        default = kw.get('default', NO_DEFAULT)
+        if default == '{}':
+            kw['default'] = {}
+            kw = compat_kwargs(kw)
+
+        return self._search_json(
+            r'''<script\s[^>]*?\bid\s*=\s*('|")__NEXT_DATA__\1[^>]*>''',
+            webpage, 'next.js data', video_id, end_pattern='</script>',
+            **kw)
 
     def _search_nuxt_data(self, webpage, video_id, *args, **kwargs):
         """Parses Nuxt.js metadata. This works as long as the function __NUXT__ invokes is a pure function"""
@@ -3296,12 +3300,16 @@ class InfoExtractor(object):
         return ret
 
     @classmethod
-    def _merge_subtitles(cls, subtitle_dict1, subtitle_dict2):
-        """ Merge two subtitle dictionaries, language by language. """
-        ret = dict(subtitle_dict1)
-        for lang in subtitle_dict2:
-            ret[lang] = cls._merge_subtitle_items(subtitle_dict1.get(lang, []), subtitle_dict2[lang])
-        return ret
+    def _merge_subtitles(cls, subtitle_dict1, *subtitle_dicts, **kwargs):
+        """ Merge subtitle dictionaries, language by language. """
+
+        # ..., * , target=None
+        target = kwargs.get('target') or dict(subtitle_dict1)
+
+        for subtitle_dict in subtitle_dicts:
+            for lang in subtitle_dict:
+                target[lang] = cls._merge_subtitle_items(target.get(lang, []), subtitle_dict[lang])
+        return target
 
     def extract_automatic_captions(self, *args, **kwargs):
         if (self._downloader.params.get('writeautomaticsub', False)
@@ -3334,6 +3342,29 @@ class InfoExtractor(object):
     def _generic_title(self, url):
         return compat_urllib_parse_unquote(os.path.splitext(url_basename(url))[0])
 
+    def _yes_playlist(self, playlist_id, video_id, *args, **kwargs):
+        # smuggled_data=None, *, playlist_label='playlist', video_label='video'
+        smuggled_data = args[0] if len(args) == 1 else kwargs.get('smuggled_data')
+        playlist_label = kwargs.get('playlist_label', 'playlist')
+        video_label = kwargs.get('video_label', 'video')
+
+        if not playlist_id or not video_id:
+            return not video_id
+
+        no_playlist = (smuggled_data or {}).get('force_noplaylist')
+        if no_playlist is not None:
+            return not no_playlist
+
+        video_id = '' if video_id is True else ' ' + video_id
+        noplaylist = self.get_param('noplaylist')
+        self.to_screen(
+            'Downloading just the {0}{1} because of --no-playlist'.format(video_label, video_id)
+            if noplaylist else
+            'Downloading {0}{1} - add --no-playlist to download just the {2}{3}'.format(
+                playlist_label, '' if playlist_id is True else ' ' + playlist_id,
+                video_label, video_id))
+        return not noplaylist
+
 
 class SearchInfoExtractor(InfoExtractor):
     """

From 88bd8b9f87f6f4956f11d32f3a7f23f20283357b Mon Sep 17 00:00:00 2001
From: kmnx <max.hampel@gmail.com>
Date: Tue, 11 Jun 2024 13:38:24 +0200
Subject: [PATCH 758/860] [mixcloud] updated mixcloud API server address
 (#32557)

* updated mixcloud API server address
* fix tests
* etc

---------

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/mixcloud.py | 30 ++++++++++++++++++++++++------
 1 file changed, 24 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/mixcloud.py b/youtube_dl/extractor/mixcloud.py
index 69319857d..2b5e2c15c 100644
--- a/youtube_dl/extractor/mixcloud.py
+++ b/youtube_dl/extractor/mixcloud.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import itertools
@@ -10,7 +11,7 @@ from ..compat import (
     compat_ord,
     compat_str,
     compat_urllib_parse_unquote,
-    compat_zip
+    compat_zip as zip,
 )
 from ..utils import (
     int_or_none,
@@ -24,7 +25,7 @@ class MixcloudBaseIE(InfoExtractor):
     def _call_api(self, object_type, object_fields, display_id, username, slug=None):
         lookup_key = object_type + 'Lookup'
         return self._download_json(
-            'https://www.mixcloud.com/graphql', display_id, query={
+            'https://app.mixcloud.com/graphql', display_id, query={
                 'query': '''{
   %s(lookup: {username: "%s"%s}) {
     %s
@@ -44,7 +45,7 @@ class MixcloudIE(MixcloudBaseIE):
             'ext': 'm4a',
             'title': 'Cryptkeeper',
             'description': 'After quite a long silence from myself, finally another Drum\'n\'Bass mix with my favourite current dance floor bangers.',
-            'uploader': 'Daniel Holbach',
+            'uploader': 'dholbach',  # was: 'Daniel Holbach',
             'uploader_id': 'dholbach',
             'thumbnail': r're:https?://.*\.jpg',
             'view_count': int,
@@ -57,7 +58,7 @@ class MixcloudIE(MixcloudBaseIE):
             'id': 'gillespeterson_caribou-7-inch-vinyl-mix-chat',
             'ext': 'mp3',
             'title': 'Caribou 7 inch Vinyl Mix & Chat',
-            'description': 'md5:2b8aec6adce69f9d41724647c65875e8',
+            'description': r're:Last week Dan Snaith aka Caribou swung by the Brownswood.{136}',
             'uploader': 'Gilles Peterson Worldwide',
             'uploader_id': 'gillespeterson',
             'thumbnail': 're:https?://.*',
@@ -65,6 +66,23 @@ class MixcloudIE(MixcloudBaseIE):
             'timestamp': 1422987057,
             'upload_date': '20150203',
         },
+        'params': {
+            'skip_download': '404 not found',
+        },
+    }, {
+        'url': 'https://www.mixcloud.com/gillespeterson/carnival-m%C3%BAsica-popular-brasileira-mix/',
+        'info_dict': {
+            'id': 'gillespeterson_carnival-música-popular-brasileira-mix',
+            'ext': 'm4a',
+            'title': 'Carnival Música Popular Brasileira Mix',
+            'description': r're:Gilles was recently in Brazil to play at Boiler Room.{208}',
+            'timestamp': 1454347174,
+            'upload_date': '20160201',
+            'uploader': 'Gilles Peterson Worldwide',
+            'uploader_id': 'gillespeterson',
+            'thumbnail': 're:https?://.*',
+            'view_count': int,
+        },
     }, {
         'url': 'https://beta.mixcloud.com/RedLightRadio/nosedrip-15-red-light-radio-01-18-2016/',
         'only_matching': True,
@@ -76,10 +94,10 @@ class MixcloudIE(MixcloudBaseIE):
         """Encrypt/Decrypt XOR cipher. Both ways are possible because it's XOR."""
         return ''.join([
             compat_chr(compat_ord(ch) ^ compat_ord(k))
-            for ch, k in compat_zip(ciphertext, itertools.cycle(key))])
+            for ch, k in zip(ciphertext, itertools.cycle(key))])
 
     def _real_extract(self, url):
-        username, slug = re.match(self._VALID_URL, url).groups()
+        username, slug = self._match_valid_url(url).groups()
         username, slug = compat_urllib_parse_unquote(username), compat_urllib_parse_unquote(slug)
         track_id = '%s_%s' % (username, slug)
 

From b4ff08bd2d12b6c91f4d8c83a7820fc6db31033d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 1 Jun 2024 03:23:37 +0100
Subject: [PATCH 759/860] [core] Safer handling of nested playlist data

---
 youtube_dl/YoutubeDL.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 6f2aba5ac..a2b45859c 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1039,8 +1039,8 @@ class YoutubeDL(object):
         elif result_type in ('playlist', 'multi_video'):
             # Protect from infinite recursion due to recursively nested playlists
             # (see https://github.com/ytdl-org/youtube-dl/issues/27833)
-            webpage_url = ie_result['webpage_url']
-            if webpage_url in self._playlist_urls:
+            webpage_url = ie_result.get('webpage_url')  # not all pl/mv have this
+            if webpage_url and webpage_url in self._playlist_urls:
                 self.to_screen(
                     '[download] Skipping already downloaded playlist: %s'
                     % ie_result.get('title') or ie_result.get('id'))
@@ -1048,6 +1048,10 @@ class YoutubeDL(object):
 
             self._playlist_level += 1
             self._playlist_urls.add(webpage_url)
+            new_result = dict((k, v) for k, v in extra_info.items() if k not in ie_result)
+            if new_result:
+                new_result.update(ie_result)
+                ie_result = new_result
             try:
                 return self.__process_playlist(ie_result, download)
             finally:

From 50f6c5668ac28a435a5c09e3d5ee7c13a50999f3 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 1 Jun 2024 03:25:06 +0100
Subject: [PATCH 760/860] [core] Re-factor with `_fill_common_fields()` as used
 in yt-dlp

---
 youtube_dl/YoutubeDL.py | 41 +++++++++++++++++++++++------------------
 1 file changed, 23 insertions(+), 18 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index a2b45859c..dad44435f 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -1597,6 +1597,28 @@ class YoutubeDL(object):
         self.cookiejar.add_cookie_header(pr)
         return pr.get_header('Cookie')
 
+    def _fill_common_fields(self, info_dict, final=True):
+
+        for ts_key, date_key in (
+                ('timestamp', 'upload_date'),
+                ('release_timestamp', 'release_date'),
+        ):
+            if info_dict.get(date_key) is None and info_dict.get(ts_key) is not None:
+                # Working around out-of-range timestamp values (e.g. negative ones on Windows,
+                # see http://bugs.python.org/issue1646728)
+                try:
+                    upload_date = datetime.datetime.utcfromtimestamp(info_dict[ts_key])
+                    info_dict[date_key] = compat_str(upload_date.strftime('%Y%m%d'))
+                except (ValueError, OverflowError, OSError):
+                    pass
+
+        # Auto generate title fields corresponding to the *_number fields when missing
+        # in order to always have clean titles. This is very common for TV series.
+        if final:
+            for field in ('chapter', 'season', 'episode'):
+                if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
+                    info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
+
     def process_video_result(self, info_dict, download=True):
         assert info_dict.get('_type', 'video') == 'video'
 
@@ -1664,24 +1686,7 @@ class YoutubeDL(object):
         if 'display_id' not in info_dict and 'id' in info_dict:
             info_dict['display_id'] = info_dict['id']
 
-        for ts_key, date_key in (
-                ('timestamp', 'upload_date'),
-                ('release_timestamp', 'release_date'),
-        ):
-            if info_dict.get(date_key) is None and info_dict.get(ts_key) is not None:
-                # Working around out-of-range timestamp values (e.g. negative ones on Windows,
-                # see http://bugs.python.org/issue1646728)
-                try:
-                    upload_date = datetime.datetime.utcfromtimestamp(info_dict[ts_key])
-                    info_dict[date_key] = compat_str(upload_date.strftime('%Y%m%d'))
-                except (ValueError, OverflowError, OSError):
-                    pass
-
-        # Auto generate title fields corresponding to the *_number fields when missing
-        # in order to always have clean titles. This is very common for TV series.
-        for field in ('chapter', 'season', 'episode'):
-            if info_dict.get('%s_number' % field) is not None and not info_dict.get(field):
-                info_dict[field] = '%s %d' % (field.capitalize(), info_dict['%s_number' % field])
+        self._fill_common_fields(info_dict)
 
         for cc_kind in ('subtitles', 'automatic_captions'):
             cc = info_dict.get(cc_kind)

From 3bde6a5752591f824096469fb9617be9d470df2c Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 11 Jun 2024 01:20:42 +0100
Subject: [PATCH 761/860] [test] Improve download test * skip reason can't be
 unicode in Py2 * remove duplicate assert...Equal functions

---
 test/helper.py        | 31 +++++--------------------------
 test/test_download.py | 22 +++++++++++++---------
 2 files changed, 18 insertions(+), 35 deletions(-)

diff --git a/test/helper.py b/test/helper.py
index 5b7e3dfe2..6f2129eff 100644
--- a/test/helper.py
+++ b/test/helper.py
@@ -5,9 +5,9 @@ import hashlib
 import json
 import os.path
 import re
-import types
 import ssl
 import sys
+import types
 import unittest
 
 import youtube_dl.extractor
@@ -181,18 +181,18 @@ def expect_value(self, got, expected, field):
             op, _, expected_num = expected.partition(':')
             expected_num = int(expected_num)
             if op == 'mincount':
-                assert_func = assertGreaterEqual
+                assert_func = self.assertGreaterEqual
                 msg_tmpl = 'Expected %d items in field %s, but only got %d'
             elif op == 'maxcount':
-                assert_func = assertLessEqual
+                assert_func = self.assertLessEqual
                 msg_tmpl = 'Expected maximum %d items in field %s, but got %d'
             elif op == 'count':
-                assert_func = assertEqual
+                assert_func = self.assertEqual
                 msg_tmpl = 'Expected exactly %d items in field %s, but got %d'
             else:
                 assert False
             assert_func(
-                self, len(got), expected_num,
+                len(got), expected_num,
                 msg_tmpl % (expected_num, field, len(got)))
             return
         self.assertEqual(
@@ -262,27 +262,6 @@ def assertRegexpMatches(self, text, regexp, msg=None):
             self.assertTrue(m, msg)
 
 
-def assertGreaterEqual(self, got, expected, msg=None):
-    if not (got >= expected):
-        if msg is None:
-            msg = '%r not greater than or equal to %r' % (got, expected)
-        self.assertTrue(got >= expected, msg)
-
-
-def assertLessEqual(self, got, expected, msg=None):
-    if not (got <= expected):
-        if msg is None:
-            msg = '%r not less than or equal to %r' % (got, expected)
-        self.assertTrue(got <= expected, msg)
-
-
-def assertEqual(self, got, expected, msg=None):
-    if not (got == expected):
-        if msg is None:
-            msg = '%r not equal to %r' % (got, expected)
-        self.assertTrue(got == expected, msg)
-
-
 def expect_warnings(ydl, warnings_re):
     real_warning = ydl.report_warning
 
diff --git a/test/test_download.py b/test/test_download.py
index df8b370cf..f7d6a23bc 100644
--- a/test/test_download.py
+++ b/test/test_download.py
@@ -9,8 +9,6 @@ import unittest
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 from test.helper import (
-    assertGreaterEqual,
-    assertLessEqual,
     expect_warnings,
     get_params,
     gettestcases,
@@ -36,12 +34,20 @@ from youtube_dl.utils import (
     ExtractorError,
     error_to_compat_str,
     format_bytes,
+    IDENTITY,
+    preferredencoding,
     UnavailableVideoError,
 )
 from youtube_dl.extractor import get_info_extractor
 
 RETRIES = 3
 
+# Some unittest APIs require actual str
+if not isinstance('TEST', str):
+    _encode_str = lambda s: s.encode(preferredencoding())
+else:
+    _encode_str = IDENTITY
+
 
 class YoutubeDL(youtube_dl.YoutubeDL):
     def __init__(self, *args, **kwargs):
@@ -102,7 +108,7 @@ def generator(test_case, tname):
 
         def print_skipping(reason):
             print('Skipping %s: %s' % (test_case['name'], reason))
-            self.skipTest(reason)
+            self.skipTest(_encode_str(reason))
 
         if not ie.working():
             print_skipping('IE marked as not _WORKING')
@@ -187,16 +193,14 @@ def generator(test_case, tname):
                 expect_info_dict(self, res_dict, test_case.get('info_dict', {}))
 
             if 'playlist_mincount' in test_case:
-                assertGreaterEqual(
-                    self,
+                self.assertGreaterEqual(
                     len(res_dict['entries']),
                     test_case['playlist_mincount'],
                     'Expected at least %d in playlist %s, but got only %d' % (
                         test_case['playlist_mincount'], test_case['url'],
                         len(res_dict['entries'])))
             if 'playlist_maxcount' in test_case:
-                assertLessEqual(
-                    self,
+                self.assertLessEqual(
                     len(res_dict['entries']),
                     test_case['playlist_maxcount'],
                     'Expected at most %d in playlist %s, but got %d' % (
@@ -243,8 +247,8 @@ def generator(test_case, tname):
                         if params.get('test'):
                             expected_minsize = max(expected_minsize, 10000)
                         got_fsize = os.path.getsize(tc_filename)
-                        assertGreaterEqual(
-                            self, got_fsize, expected_minsize,
+                        self.assertGreaterEqual(
+                            got_fsize, expected_minsize,
                             'Expected %s to be at least %s, but it\'s only %s ' %
                             (tc_filename, format_bytes(expected_minsize),
                                 format_bytes(got_fsize)))

From d95c0d203fa74c806a806806f7c5286ca532b936 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 1 Jun 2024 12:56:41 +0100
Subject: [PATCH 762/860] [ORF] Support on.orf.at, replacing `ORFTVthekIE` *
 add `ORFONIE`, back-porting yt-dlp PR
 https://github.com/yt-dlp/yt-dlp/pull/9113 and friends: thx HobbyistDev,
 TuxCoder, seproDev * re-factor to support livestreams via new `ORFONliveIE`

---
 youtube_dl/extractor/extractors.py |   3 +-
 youtube_dl/extractor/orf.py        | 434 ++++++++++++++++++-----------
 2 files changed, 274 insertions(+), 163 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 03d035a27..8bc5dd6d4 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -898,7 +898,8 @@ from .ooyala import (
 )
 from .ora import OraTVIE
 from .orf import (
-    ORFTVthekIE,
+    ORFONIE,
+    ORFONLiveIE,
     ORFFM4IE,
     ORFFM4StoryIE,
     ORFOE1IE,
diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 8d537d7ae..3854911bd 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -1,184 +1,30 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import base64
+import functools
 import re
 
 from .common import InfoExtractor
-from ..compat import compat_str
 from ..utils import (
     clean_html,
     determine_ext,
     float_or_none,
-    HEADRequest,
     int_or_none,
+    merge_dicts,
     orderedSet,
+    parse_age_limit,
+    parse_iso8601,
     remove_end,
     str_or_none,
     strip_jsonp,
-    unescapeHTML,
+    txt_or_none,
     unified_strdate,
     url_or_none,
 )
+from ..traversal import T, traverse_obj
 
-
-class ORFTVthekIE(InfoExtractor):
-    IE_NAME = 'orf:tvthek'
-    IE_DESC = 'ORF TVthek'
-    _VALID_URL = r'https?://tvthek\.orf\.at/(?:[^/]+/)+(?P<id>\d+)'
-
-    _TESTS = [{
-        'url': 'http://tvthek.orf.at/program/Aufgetischt/2745173/Aufgetischt-Mit-der-Steirischen-Tafelrunde/8891389',
-        'playlist': [{
-            'md5': '2942210346ed779588f428a92db88712',
-            'info_dict': {
-                'id': '8896777',
-                'ext': 'mp4',
-                'title': 'Aufgetischt: Mit der Steirischen Tafelrunde',
-                'description': 'md5:c1272f0245537812d4e36419c207b67d',
-                'duration': 2668,
-                'upload_date': '20141208',
-            },
-        }],
-        'skip': 'Blocked outside of Austria / Germany',
-    }, {
-        'url': 'http://tvthek.orf.at/topic/Im-Wandel-der-Zeit/8002126/Best-of-Ingrid-Thurnher/7982256',
-        'info_dict': {
-            'id': '7982259',
-            'ext': 'mp4',
-            'title': 'Best of Ingrid Thurnher',
-            'upload_date': '20140527',
-            'description': 'Viele Jahre war Ingrid Thurnher das "Gesicht" der ZIB 2. Vor ihrem Wechsel zur ZIB 2 im Jahr 1995 moderierte sie unter anderem "Land und Leute", "Österreich-Bild" und "Niederösterreich heute".',
-        },
-        'params': {
-            'skip_download': True,  # rtsp downloads
-        },
-        'skip': 'Blocked outside of Austria / Germany',
-    }, {
-        'url': 'http://tvthek.orf.at/topic/Fluechtlingskrise/10463081/Heimat-Fremde-Heimat/13879132/Senioren-betreuen-Migrantenkinder/13879141',
-        'only_matching': True,
-    }, {
-        'url': 'http://tvthek.orf.at/profile/Universum/35429',
-        'only_matching': True,
-    }]
-
-    def _real_extract(self, url):
-        playlist_id = self._match_id(url)
-        webpage = self._download_webpage(url, playlist_id)
-
-        data_jsb = self._parse_json(
-            self._search_regex(
-                r'<div[^>]+class=(["\']).*?VideoPlaylist.*?\1[^>]+data-jsb=(["\'])(?P<json>.+?)\2',
-                webpage, 'playlist', group='json'),
-            playlist_id, transform_source=unescapeHTML)['playlist']['videos']
-
-        entries = []
-        for sd in data_jsb:
-            video_id, title = sd.get('id'), sd.get('title')
-            if not video_id or not title:
-                continue
-            video_id = compat_str(video_id)
-            formats = []
-            for fd in sd['sources']:
-                src = url_or_none(fd.get('src'))
-                if not src:
-                    continue
-                format_id_list = []
-                for key in ('delivery', 'quality', 'quality_string'):
-                    value = fd.get(key)
-                    if value:
-                        format_id_list.append(value)
-                format_id = '-'.join(format_id_list)
-                ext = determine_ext(src)
-                if ext == 'm3u8':
-                    m3u8_formats = self._extract_m3u8_formats(
-                        src, video_id, 'mp4', m3u8_id=format_id, fatal=False)
-                    if any('/geoprotection' in f['url'] for f in m3u8_formats):
-                        self.raise_geo_restricted()
-                    formats.extend(m3u8_formats)
-                elif ext == 'f4m':
-                    formats.extend(self._extract_f4m_formats(
-                        src, video_id, f4m_id=format_id, fatal=False))
-                elif ext == 'mpd':
-                    formats.extend(self._extract_mpd_formats(
-                        src, video_id, mpd_id=format_id, fatal=False))
-                else:
-                    formats.append({
-                        'format_id': format_id,
-                        'url': src,
-                        'protocol': fd.get('protocol'),
-                    })
-
-            # Check for geoblocking.
-            # There is a property is_geoprotection, but that's always false
-            geo_str = sd.get('geoprotection_string')
-            if geo_str:
-                try:
-                    http_url = next(
-                        f['url']
-                        for f in formats
-                        if re.match(r'^https?://.*\.mp4$', f['url']))
-                except StopIteration:
-                    pass
-                else:
-                    req = HEADRequest(http_url)
-                    self._request_webpage(
-                        req, video_id,
-                        note='Testing for geoblocking',
-                        errnote=((
-                            'This video seems to be blocked outside of %s. '
-                            'You may want to try the streaming-* formats.')
-                            % geo_str),
-                        fatal=False)
-
-            self._check_formats(formats, video_id)
-            self._sort_formats(formats)
-
-            subtitles = {}
-            for sub in sd.get('subtitles', []):
-                sub_src = sub.get('src')
-                if not sub_src:
-                    continue
-                subtitles.setdefault(sub.get('lang', 'de-AT'), []).append({
-                    'url': sub_src,
-                })
-
-            upload_date = unified_strdate(sd.get('created_date'))
-
-            thumbnails = []
-            preview = sd.get('preview_image_url')
-            if preview:
-                thumbnails.append({
-                    'id': 'preview',
-                    'url': preview,
-                    'preference': 0,
-                })
-            image = sd.get('image_full_url')
-            if not image and len(data_jsb) == 1:
-                image = self._og_search_thumbnail(webpage)
-            if image:
-                thumbnails.append({
-                    'id': 'full',
-                    'url': image,
-                    'preference': 1,
-                })
-
-            entries.append({
-                '_type': 'video',
-                'id': video_id,
-                'title': title,
-                'formats': formats,
-                'subtitles': subtitles,
-                'description': sd.get('description'),
-                'duration': int_or_none(sd.get('duration_in_seconds')),
-                'upload_date': upload_date,
-                'thumbnails': thumbnails,
-            })
-
-        return {
-            '_type': 'playlist',
-            'entries': entries,
-            'id': playlist_id,
-        }
+k_float_or_none = functools.partial(float_or_none, scale=1000)
 
 
 class ORFRadioIE(InfoExtractor):
@@ -401,6 +247,7 @@ class ORFOE1IE(ORFRadioIE):
 class ORFIPTVIE(InfoExtractor):
     IE_NAME = 'orf:iptv'
     IE_DESC = 'iptv.ORF.at'
+    _WORKING = False  # URLs redirect to orf.at/
     _VALID_URL = r'https?://iptv\.orf\.at/(?:#/)?stories/(?P<id>\d+)'
 
     _TEST = {
@@ -590,3 +437,266 @@ class ORFFM4StoryIE(InfoExtractor):
             })
 
         return self.playlist_result(entries)
+
+
+class ORFONBase(InfoExtractor):
+    _ENC_PFX = '3dSlfek03nsLKdj4Jsd'
+    _API_PATH = 'episode'
+
+    def _call_api(self, video_id, **kwargs):
+        encrypted_id = base64.b64encode('{0}{1}'.format(
+            self._ENC_PFX, video_id).encode('utf-8')).decode('ascii')
+        return self._download_json(
+            'https://api-tvthek.orf.at/api/v4.3/public/{0}/encrypted/{1}'.format(
+                self._API_PATH, encrypted_id),
+            video_id, **kwargs)
+
+    @classmethod
+    def _parse_metadata(cls, api_json):
+        return traverse_obj(api_json, {
+            'id': ('id', T(int), T(txt_or_none)),
+            'age_limit': ('age_classification', T(parse_age_limit)),
+            'duration': ((('exact_duration', T(k_float_or_none)),
+                          ('duration_second', T(float_or_none))),),
+            'title': (('title', 'headline'), T(txt_or_none)),
+            'description': (('description', 'teaser_text'), T(txt_or_none)),
+            # 'media_type': ('video_type', T(txt_or_none)),
+            'thumbnail': ('_embedded', 'image', 'public_urls', 'highlight_teaser', 'url', T(url_or_none)),
+            'timestamp': (('date', 'episode_date'), T(parse_iso8601)),
+            'release_timestamp': ('release_date', T(parse_iso8601)),
+            # 'modified_timestamp': ('updated_at', T(parse_iso8601)),
+        }, get_all=False)
+
+    def _extract_video(self, video_id, segment_id):
+        # Not a segmented episode: return single video
+        # Segmented episode without valid segment id: return entire playlist
+        # Segmented episode with valid segment id and yes-playlist: return entire playlist
+        # Segmented episode with valid segment id and no-playlist: return single video corresponding to segment id
+        # If a multi_video playlist would be returned, but an unsegmented source exists, that source is chosen instead.
+
+        api_json = self._call_api(video_id)
+
+        if traverse_obj(api_json, 'is_drm_protected'):
+            self.report_drm(video_id)
+
+        # updates formats, subtitles
+        def extract_sources(src_json, video_id):
+            for manifest_type in traverse_obj(src_json, ('sources', T(dict.keys), Ellipsis)):
+                for manifest_url in traverse_obj(src_json, ('sources', manifest_type, Ellipsis, 'src', T(url_or_none))):
+                    if manifest_type == 'hls':
+                        fmts, subs = self._extract_m3u8_formats(
+                            manifest_url, video_id, fatal=False, m3u8_id='hls',
+                            ext='mp4', entry_protocol='m3u8_native'), {}
+                        for f in fmts:
+                            if '_vo.' in f['url']:
+                                f['acodec'] = 'none'
+                    elif manifest_type == 'dash':
+                        fmts, subs = self._extract_mpd_formats_and_subtitles(
+                            manifest_url, video_id, fatal=False, mpd_id='dash')
+                    else:
+                        continue
+                    formats.extend(fmts)
+                    self._merge_subtitles(subs, target=subtitles)
+
+        formats, subtitles = [], {}
+        if segment_id is None:
+            extract_sources(api_json, video_id)
+        if not formats:
+            segments = traverse_obj(api_json, (
+                '_embedded', 'segments', lambda _, v: v['id']))
+            if len(segments) > 1 and segment_id is not None:
+                if not self._yes_playlist(video_id, segment_id, playlist_label='collection', video_label='segment'):
+                    segments = [next(s for s in segments if txt_or_none(s['id']) == segment_id)]
+
+            entries = []
+            for seg in segments:
+                formats, subtitles = [], {}
+                extract_sources(seg, segment_id)
+                self._sort_formats(formats)
+                entries.append(merge_dicts({
+                    'formats': formats,
+                    'subtitles': subtitles,
+                }, self._parse_metadata(seg), rev=True))
+            result = merge_dicts(
+                {'_type': 'multi_video' if len(entries) > 1 else 'playlist'},
+                self._parse_metadata(api_json),
+                self.playlist_result(entries, video_id))
+            # not yet processed in core for playlist/multi
+            self._downloader._fill_common_fields(result)
+            return result
+        else:
+            self._sort_formats(formats)
+
+        for sub_url in traverse_obj(api_json, (
+                '_embedded', 'subtitle',
+                ('xml_url', 'sami_url', 'stl_url', 'ttml_url', 'srt_url', 'vtt_url'),
+                T(url_or_none))):
+            self._merge_subtitles({'de': [{'url': sub_url}]}, target=subtitles)
+
+        return merge_dicts({
+            'id': video_id,
+            'formats': formats,
+            'subtitles': subtitles,
+            # '_old_archive_ids': [self._downloader._make_archive_id({'ie_key': 'ORFTVthek', 'id': video_id})],
+        }, self._parse_metadata(api_json), rev=True)
+
+    def _real_extract(self, url):
+        video_id, segment_id = self._match_valid_url(url).group('id', 'segment')
+        webpage = self._download_webpage(url, video_id)
+
+        # ORF doesn't like 410 or 404
+        if self._search_regex(r'<div\b[^>]*>\s*(Nicht mehr verfügbar)\s*</div>', webpage, 'Availability', default=False):
+            raise ExtractorError('Content is no longer available', expected=True, video_id=video_id)
+
+        return merge_dicts({
+            'id': video_id,
+            'title': self._html_search_meta(['og:title', 'twitter:title'], webpage, default=None),
+            'description': self._html_search_meta(
+                ['description', 'og:description', 'twitter:description'], webpage, default=None),
+        }, self._search_json_ld(webpage, video_id, default={}),
+            self._extract_video(video_id, segment_id),
+            rev=True)
+
+
+class ORFONIE(ORFONBase):
+    IE_NAME = 'orf:on'
+    _VALID_URL = r'https?://on\.orf\.at/video/(?P<id>\d+)(?:/(?P<segment>\d+))?'
+    _TESTS = [{
+        'url': 'https://on.orf.at/video/14210000/school-of-champions-48',
+        'info_dict': {
+            'id': '14210000',
+            'ext': 'mp4',
+            'duration': 2651.08,
+            'thumbnail': 'https://api-tvthek.orf.at/assets/segments/0167/98/thumb_16697671_segments_highlight_teaser.jpeg',
+            'title': 'School of Champions (4/8)',
+            'description': r're:(?s)Luca hat sein ganzes Leben in den Bergen Südtirols verbracht und ist bei seiner Mutter aufgewachsen, .{1029} Leo$',
+            # 'media_type': 'episode',
+            'timestamp': 1706558922,
+            'upload_date': '20240129',
+            'release_timestamp': 1706472362,
+            'release_date': '20240128',
+            # 'modified_timestamp': 1712756663,
+            # 'modified_date': '20240410',
+            # '_old_archive_ids': ['orftvthek 14210000'],
+        },
+        'params': {
+            'format': 'bestvideo',
+        },
+    }, {
+        'url': 'https://on.orf.at/video/3220355',
+        'md5': '925a93b2b9a37da5c9b979d7cf71aa2e',
+        'info_dict': {
+            'id': '3220355',
+            'ext': 'mp4',
+            'duration': 445.04,
+            'thumbnail': 'https://api-tvthek.orf.at/assets/segments/0002/60/thumb_159573_segments_highlight_teaser.png',
+            'title': '50 Jahre Burgenland: Der Festumzug',
+            'description': r're:(?s)Aus allen Landesteilen zogen festlich geschmückte Wagen und Musikkapellen .{270} Jenakowitsch$',
+            # 'media_type': 'episode',
+            'timestamp': 52916400,
+            'upload_date': '19710905',
+            'release_timestamp': 52916400,
+            'release_date': '19710905',
+            # 'modified_timestamp': 1498536049,
+            # 'modified_date': '20170627',
+            # '_old_archive_ids': ['orftvthek 3220355'],
+        },
+    }, {
+        # Video with multiple segments selecting the second segment
+        'url': 'https://on.orf.at/video/14226549/15639808/jugendbande-einbrueche-aus-langeweile',
+        'md5': 'fc151bba8c05ea77ab5693617e4a33d3',
+        'info_dict': {
+            'id': '15639808',
+            'ext': 'mp4',
+            'duration': 97.707,
+            'thumbnail': 'https://api-tvthek.orf.at/assets/segments/0175/43/thumb_17442704_segments_highlight_teaser.jpg',
+            'title': 'Jugendbande: Einbrüche aus Langeweile',
+            'description': r're:Jugendbande: Einbrüche aus Langeweile \| Neuer Kinder- und .{259} Wanda$',
+            # 'media_type': 'segment',
+            'timestamp': 1715792400,
+            'upload_date': '20240515',
+            # 'modified_timestamp': 1715794394,
+            # 'modified_date': '20240515',
+            # '_old_archive_ids': ['orftvthek 15639808'],
+        },
+        'params': {
+            'noplaylist': True,
+            'format': 'bestvideo',
+        },
+    }, {
+        # Video with multiple segments and no combined version
+        'url': 'https://on.orf.at/video/14227864/formel-1-grosser-preis-von-monaco-2024',
+        'info_dict': {
+            '_type': 'multi_video',
+            'id': '14227864',
+            'duration': 18410.52,
+            'thumbnail': 'https://api-tvthek.orf.at/assets/segments/0176/04/thumb_17503881_segments_highlight_teaser.jpg',
+            'title': 'Formel 1: Großer Preis von Monaco 2024',
+            'description': 'md5:aeeb010710ccf70ce28ccb4482243d4f',
+            # 'media_type': 'episode',
+            'timestamp': 1716721200,
+            'upload_date': '20240526',
+            'release_timestamp': 1716721802,
+            'release_date': '20240526',
+            # 'modified_timestamp': 1716884702,
+            # 'modified_date': '20240528',
+        },
+        'playlist_count': 42,
+        'skip': 'Gone: Nicht mehr verfügbar',
+    }, {
+        # Video with multiple segments, but with combined version
+        'url': 'https://on.orf.at/video/14228172',
+        'info_dict': {
+            'id': '14228172',
+            'ext': 'mp4',
+            'duration': 3294.878,
+            'thumbnail': 'https://api-tvthek.orf.at/assets/segments/0176/29/thumb_17528242_segments_highlight_teaser.jpg',
+            'title': 'Willkommen Österreich mit Stermann & Grissemann',
+            'description': r're:Zum Saisonfinale freuen sich die urlaubsreifen Gastgeber Stermann und .{1863} Geschichten\.$',
+            # 'media_type': 'episode',
+            'timestamp': 1716926584,
+            'upload_date': '20240528',
+            'release_timestamp': 1716919202,
+            'release_date': '20240528',
+            # 'modified_timestamp': 1716968045,
+            # 'modified_date': '20240529',
+            # '_old_archive_ids': ['orftvthek 14228172'],
+        },
+        'params': {
+            'format': 'bestvideo',
+        },
+        'skip': 'Gone: Nicht mehr verfügbar',
+    }]
+
+
+class ORFONLiveIE(ORFONBase):
+    _ENC_PFX = '8876324jshjd7293ktd'
+    _API_PATH = 'livestream'
+    _VALID_URL = r'https?://on\.orf\.at/livestream/(?P<id>\d+)(?:/(?P<segment>\d+))?'
+    _TESTS = [{
+        'url': 'https://on.orf.at/livestream/14320204/pressekonferenz-neos-zu-aktuellen-entwicklungen',
+        'info_dict': {
+            'id': '14320204',
+            'ext': 'mp4',
+            'title': 'Pressekonferenz: Neos zu aktuellen Entwicklungen',
+            'description': r're:(?s)Neos-Chefin Beate Meinl-Reisinger informi.{598}ng\."',
+            'timestamp': 1716886335,
+            'upload_date': '20240528',
+            # 'modified_timestamp': 1712756663,
+            # 'modified_date': '20240410',
+            # '_old_archive_ids': ['orftvthek 14210000'],
+        },
+        'params': {
+            'format': 'bestvideo',
+        },
+    }]
+
+    @classmethod
+    def _parse_metadata(cls, api_json):
+        return merge_dicts(
+            super(ORFONLiveIE, cls)._parse_metadata(api_json),
+            traverse_obj(api_json, {
+                'timestamp': ('updated_at', T(parse_iso8601)),
+                'release_timestamp': ('start', T(parse_iso8601)),
+                'is_live': True,
+            }))

From e39466051f01411944bd657fe826b658a0df5af1 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 1 Jun 2024 13:29:26 +0100
Subject: [PATCH 763/860] [ORF] Support sound.orf.at, updating `ORFRadioIE` *
 maintain support for xx.orf.at/player/... URLs * add `ORFRadioCollectionIE`
 to support playlists in ORF Sound * back-port and re-work `ORFPodcastIE` from
 https://github.com/yt-dlp/yt-dlp/pull/8486, thx Esokrates

---
 youtube_dl/extractor/extractors.py |  15 +-
 youtube_dl/extractor/orf.py        | 469 ++++++++++++++++-------------
 2 files changed, 268 insertions(+), 216 deletions(-)

diff --git a/youtube_dl/extractor/extractors.py b/youtube_dl/extractor/extractors.py
index 8bc5dd6d4..3da5f8020 100644
--- a/youtube_dl/extractor/extractors.py
+++ b/youtube_dl/extractor/extractors.py
@@ -900,20 +900,11 @@ from .ora import OraTVIE
 from .orf import (
     ORFONIE,
     ORFONLiveIE,
-    ORFFM4IE,
     ORFFM4StoryIE,
-    ORFOE1IE,
-    ORFOE3IE,
-    ORFNOEIE,
-    ORFWIEIE,
-    ORFBGLIE,
-    ORFOOEIE,
-    ORFSTMIE,
-    ORFKTNIE,
-    ORFSBGIE,
-    ORFTIRIE,
-    ORFVBGIE,
     ORFIPTVIE,
+    ORFPodcastIE,
+    ORFRadioIE,
+    ORFRadioCollectionIE,
 )
 from .outsidetv import OutsideTVIE
 from .packtpub import (
diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 3854911bd..25c16c84d 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -9,17 +9,19 @@ from .common import InfoExtractor
 from ..utils import (
     clean_html,
     determine_ext,
+    ExtractorError,
     float_or_none,
     int_or_none,
     merge_dicts,
+    mimetype2ext,
     orderedSet,
     parse_age_limit,
     parse_iso8601,
     remove_end,
-    str_or_none,
     strip_jsonp,
     txt_or_none,
     unified_strdate,
+    update_url_query,
     url_or_none,
 )
 from ..traversal import T, traverse_obj
@@ -27,221 +29,280 @@ from ..traversal import T, traverse_obj
 k_float_or_none = functools.partial(float_or_none, scale=1000)
 
 
-class ORFRadioIE(InfoExtractor):
-    def _real_extract(self, url):
-        mobj = re.match(self._VALID_URL, url)
-        show_date = mobj.group('date')
-        show_id = mobj.group('show')
+class ORFRadioBase(InfoExtractor):
+    STATION_INFO = {
+        'fm4': ('fm4', 'fm4', 'orffm4'),
+        'noe': ('noe', 'oe2n', 'orfnoe'),
+        'wien': ('wie', 'oe2w', 'orfwie'),
+        'burgenland': ('bgl', 'oe2b', 'orfbgl'),
+        'ooe': ('ooe', 'oe2o', 'orfooe'),
+        'steiermark': ('stm', 'oe2st', 'orfstm'),
+        'kaernten': ('ktn', 'oe2k', 'orfktn'),
+        'salzburg': ('sbg', 'oe2s', 'orfsbg'),
+        'tirol': ('tir', 'oe2t', 'orftir'),
+        'vorarlberg': ('vbg', 'oe2v', 'orfvbg'),
+        'oe3': ('oe3', 'oe3', 'orfoe3'),
+        'oe1': ('oe1', 'oe1', 'orfoe1'),
+    }
+    _ID_NAMES = ('id', 'guid', 'program')
 
-        data = self._download_json(
-            'http://audioapi.orf.at/%s/api/json/current/broadcast/%s/%s'
-            % (self._API_STATION, show_id, show_date), show_id)
+    @classmethod
+    def _get_item_id(cls, data):
+        return traverse_obj(data, *cls._ID_NAMES, expected_type=txt_or_none)
 
-        entries = []
-        for info in data['streams']:
-            loop_stream_id = str_or_none(info.get('loopStreamId'))
-            if not loop_stream_id:
-                continue
-            title = str_or_none(data.get('title'))
-            if not title:
-                continue
-            start = int_or_none(info.get('start'), scale=1000)
-            end = int_or_none(info.get('end'), scale=1000)
-            duration = end - start if end and start else None
-            entries.append({
-                'id': loop_stream_id.replace('.mp3', ''),
-                'url': 'https://loopstream01.apa.at/?channel=%s&id=%s' % (self._LOOP_STATION, loop_stream_id),
-                'title': title,
-                'description': clean_html(data.get('subtitle')),
-                'duration': duration,
-                'timestamp': start,
+    @classmethod
+    def _get_api_payload(cls, data, expected_id, in_payload=False):
+        if expected_id not in traverse_obj(data, ('payload',)[:1 if in_payload else 0] + (cls._ID_NAMES, T(txt_or_none))):
+            raise ExtractorError('Unexpected API data result', video_id=expected_id)
+        return data['payload']
+
+    @staticmethod
+    def _extract_podcast_upload(data):
+        return traverse_obj(data, {
+            'url': ('enclosures', 0, 'url'),
+            'ext': ('enclosures', 0, 'type', T(mimetype2ext)),
+            'filesize': ('enclosures', 0, 'length', T(int_or_none)),
+            'title': ('title', T(txt_or_none)),
+            'description': ('description', T(clean_html)),
+            'timestamp': (('published', 'postDate'), T(parse_iso8601)),
+            'duration': ('duration', T(k_float_or_none)),
+            'series': ('podcast', 'title'),
+            'uploader': ((('podcast', 'author'), 'station'), T(txt_or_none)),
+            'uploader_id': ('podcast', 'channel', T(txt_or_none)),
+        }, get_all=False)
+
+    @classmethod
+    def _entries(cls, data, station, item_type=None):
+        if item_type in ('upload', 'podcast-episode'):
+            yield merge_dicts({
+                'id': cls._get_item_id(data),
                 'ext': 'mp3',
-                'series': data.get('programTitle'),
-            })
+                'vcodec': 'none',
+            }, cls._extract_podcast_upload(data), rev=True)
+            return
 
-        return {
-            '_type': 'playlist',
-            'id': show_id,
-            'title': data.get('title'),
-            'description': clean_html(data.get('subtitle')),
-            'entries': entries,
-        }
+        loop_station = cls.STATION_INFO[station][1]
+        for info in traverse_obj(data, ((('streams', Ellipsis), 'stream'), T(lambda v: v if v['loopStreamId'] else None))):
+            item_id = info['loopStreamId']
+            host = info.get('host') or 'loopstream01.apa.at'
+            yield merge_dicts({
+                'id': item_id.replace('.mp3', ''),
+                'ext': 'mp3',
+                'url': update_url_query('https://{0}/'.format(host), {
+                    'channel': loop_station,
+                    'id': item_id,
+                }),
+                'vcodec': 'none',
+                # '_old_archive_ids': [make_archive_id(old_ie, video_id)],
+            }, traverse_obj(data, {
+                'title': ('title', T(txt_or_none)),
+                'description': ('subtitle', T(clean_html)),
+                'uploader': 'station',
+                'series': ('programTitle', T(txt_or_none)),
+            }), traverse_obj(info, {
+                'duration': (('duration',
+                              (None, T(lambda x: x['end'] - x['start']))),
+                             T(k_float_or_none), any),
+                'timestamp': (('start', 'startISO'), T(parse_iso8601), any),
+            }))
 
 
-class ORFFM4IE(ORFRadioIE):
-    IE_NAME = 'orf:fm4'
-    IE_DESC = 'radio FM4'
-    _VALID_URL = r'https?://(?P<station>fm4)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>4\w+)'
-    _API_STATION = 'fm4'
-    _LOOP_STATION = 'fm4'
+class ORFRadioIE(ORFRadioBase):
+    IE_NAME = 'orf:sound'
+    _STATION_RE = '|'.join(map(re.escape, ORFRadioBase.STATION_INFO.keys()))
 
-    _TEST = {
-        'url': 'http://fm4.orf.at/player/20170107/4CC',
-        'md5': '2b0be47375432a7ef104453432a19212',
+    _VALID_URL = (
+        r'https?://sound\.orf\.at/radio/(?P<station>{0})/sendung/(?P<id>\d+)(?:/(?P<show>\w+))?'.format(_STATION_RE),
+        r'https?://(?P<station>{0})\.orf\.at/player/(?P<date>\d{{8}})/(?P<id>\d+)'.format(_STATION_RE),
+    )
+
+    _TESTS = [{
+        'url': 'https://sound.orf.at/radio/ooe/sendung/37802/guten-morgen-oberoesterreich-am-feiertag',
         'info_dict': {
-            'id': '2017-01-07_2100_tl_54_7DaysSat18_31295',
-            'ext': 'mp3',
-            'title': 'Solid Steel Radioshow',
-            'description': 'Die Mixshow von Coldcut und Ninja Tune.',
-            'duration': 3599,
-            'timestamp': 1483819257,
-            'upload_date': '20170107',
+            'id': '37802',
+            'title': 'Guten Morgen Oberösterreich am Feiertag',
+            'description': 'Oberösterreichs meistgehörte regionale Frühsendung.\nRegionale Nachrichten zu jeder halben Stunde.\nModeration: Wolfgang Lehner\nNachrichten:  Stephan Schnabl',
         },
-        'skip': 'Shows from ORF radios are only available for 7 days.',
-        'only_matching': True,
-    }
-
-
-class ORFNOEIE(ORFRadioIE):
-    IE_NAME = 'orf:noe'
-    IE_DESC = 'Radio Niederösterreich'
-    _VALID_URL = r'https?://(?P<station>noe)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
-    _API_STATION = 'noe'
-    _LOOP_STATION = 'oe2n'
-
-    _TEST = {
-        'url': 'https://noe.orf.at/player/20200423/NGM',
-        'only_matching': True,
-    }
-
-
-class ORFWIEIE(ORFRadioIE):
-    IE_NAME = 'orf:wien'
-    IE_DESC = 'Radio Wien'
-    _VALID_URL = r'https?://(?P<station>wien)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
-    _API_STATION = 'wie'
-    _LOOP_STATION = 'oe2w'
-
-    _TEST = {
-        'url': 'https://wien.orf.at/player/20200423/WGUM',
-        'only_matching': True,
-    }
-
-
-class ORFBGLIE(ORFRadioIE):
-    IE_NAME = 'orf:burgenland'
-    IE_DESC = 'Radio Burgenland'
-    _VALID_URL = r'https?://(?P<station>burgenland)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
-    _API_STATION = 'bgl'
-    _LOOP_STATION = 'oe2b'
-
-    _TEST = {
-        'url': 'https://burgenland.orf.at/player/20200423/BGM',
-        'only_matching': True,
-    }
-
-
-class ORFOOEIE(ORFRadioIE):
-    IE_NAME = 'orf:oberoesterreich'
-    IE_DESC = 'Radio Oberösterreich'
-    _VALID_URL = r'https?://(?P<station>ooe)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
-    _API_STATION = 'ooe'
-    _LOOP_STATION = 'oe2o'
-
-    _TEST = {
-        'url': 'https://ooe.orf.at/player/20200423/OGMO',
-        'only_matching': True,
-    }
-
-
-class ORFSTMIE(ORFRadioIE):
-    IE_NAME = 'orf:steiermark'
-    IE_DESC = 'Radio Steiermark'
-    _VALID_URL = r'https?://(?P<station>steiermark)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
-    _API_STATION = 'stm'
-    _LOOP_STATION = 'oe2st'
-
-    _TEST = {
-        'url': 'https://steiermark.orf.at/player/20200423/STGMS',
-        'only_matching': True,
-    }
-
-
-class ORFKTNIE(ORFRadioIE):
-    IE_NAME = 'orf:kaernten'
-    IE_DESC = 'Radio Kärnten'
-    _VALID_URL = r'https?://(?P<station>kaernten)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
-    _API_STATION = 'ktn'
-    _LOOP_STATION = 'oe2k'
-
-    _TEST = {
-        'url': 'https://kaernten.orf.at/player/20200423/KGUMO',
-        'only_matching': True,
-    }
-
-
-class ORFSBGIE(ORFRadioIE):
-    IE_NAME = 'orf:salzburg'
-    IE_DESC = 'Radio Salzburg'
-    _VALID_URL = r'https?://(?P<station>salzburg)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
-    _API_STATION = 'sbg'
-    _LOOP_STATION = 'oe2s'
-
-    _TEST = {
-        'url': 'https://salzburg.orf.at/player/20200423/SGUM',
-        'only_matching': True,
-    }
-
-
-class ORFTIRIE(ORFRadioIE):
-    IE_NAME = 'orf:tirol'
-    IE_DESC = 'Radio Tirol'
-    _VALID_URL = r'https?://(?P<station>tirol)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
-    _API_STATION = 'tir'
-    _LOOP_STATION = 'oe2t'
-
-    _TEST = {
-        'url': 'https://tirol.orf.at/player/20200423/TGUMO',
-        'only_matching': True,
-    }
-
-
-class ORFVBGIE(ORFRadioIE):
-    IE_NAME = 'orf:vorarlberg'
-    IE_DESC = 'Radio Vorarlberg'
-    _VALID_URL = r'https?://(?P<station>vorarlberg)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
-    _API_STATION = 'vbg'
-    _LOOP_STATION = 'oe2v'
-
-    _TEST = {
-        'url': 'https://vorarlberg.orf.at/player/20200423/VGUM',
-        'only_matching': True,
-    }
-
-
-class ORFOE3IE(ORFRadioIE):
-    IE_NAME = 'orf:oe3'
-    IE_DESC = 'Radio Österreich 3'
-    _VALID_URL = r'https?://(?P<station>oe3)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
-    _API_STATION = 'oe3'
-    _LOOP_STATION = 'oe3'
-
-    _TEST = {
-        'url': 'https://oe3.orf.at/player/20200424/3WEK',
-        'only_matching': True,
-    }
-
-
-class ORFOE1IE(ORFRadioIE):
-    IE_NAME = 'orf:oe1'
-    IE_DESC = 'Radio Österreich 1'
-    _VALID_URL = r'https?://(?P<station>oe1)\.orf\.at/player/(?P<date>[0-9]+)/(?P<show>\w+)'
-    _API_STATION = 'oe1'
-    _LOOP_STATION = 'oe1'
-
-    _TEST = {
-        'url': 'http://oe1.orf.at/player/20170108/456544',
-        'md5': '34d8a6e67ea888293741c86a099b745b',
+        'playlist': [{
+            'md5': 'f9ff8517dd681b642a2c900e2c9e6085',
+            'info_dict': {
+                'id': '2024-05-30_0559_tl_66_7DaysThu1_443862',
+                'ext': 'mp3',
+                'title': 'Guten Morgen Oberösterreich am Feiertag',
+                'description': 'Oberösterreichs meistgehörte regionale Frühsendung.\nRegionale Nachrichten zu jeder halben Stunde.\nModeration: Wolfgang Lehner\nNachrichten:  Stephan Schnabl',
+                'timestamp': 1717041587,
+                'upload_date': '20240530',
+                'uploader': 'ooe',
+                'duration': 14413.0,
+            }
+        }],
+        # 'skip': 'Shows from ORF Sound are only available for 30 days.'
+    }, {
+        'url': 'https://oe1.orf.at/player/20240531/758136',
+        'md5': '2397717aaf3ae9c22a4f090ee3b8d374',
         'info_dict': {
-            'id': '2017-01-08_0759_tl_51_7DaysSun6_256141',
+            'id': '2024-05-31_1905_tl_51_7DaysFri35_2413387',
             'ext': 'mp3',
-            'title': 'Morgenjournal',
-            'duration': 609,
-            'timestamp': 1483858796,
-            'upload_date': '20170108',
+            'title': '"Who Cares?"',
+            'description': 'Europas größte Netzkonferenz re:publica 2024',
+            'timestamp': 1717175100,
+            'upload_date': '20240531',
+            'uploader': 'oe1',
+            'duration': 1500,
         },
-        'skip': 'Shows from ORF radios are only available for 7 days.'
-    }
+        # 'skip': 'Shows from ORF Sound are only available for 30 days.'
+    }]
+
+    def _real_extract(self, url):
+        m = self._match_valid_url(url)
+        station, show_id = m.group('station', 'id')
+        api_station, _, _ = self.STATION_INFO[station]
+        if 'date' in m.groupdict():
+            data = self._download_json(
+                'https://audioapi.orf.at/{0}/json/4.0/broadcast/{1}/{2}?_o={3}.orf.at'.format(
+                    api_station, show_id, m.group('date'), station), show_id)
+            show_id = data['id']
+        else:
+            data = self._download_json(
+                'https://audioapi.orf.at/{0}/api/json/5.0/broadcast/{1}?_o=sound.orf.at'.format(
+                    api_station, show_id), show_id)
+
+            data = self._get_api_payload(data, show_id, in_payload=True)
+
+        # site sends ISO8601 GMT date-times with separate TZ offset, ignored
+        # TODO: should `..._date` be calculated relative to TZ?
+
+        return merge_dicts(
+            {'_type': 'multi_video'},
+            self.playlist_result(
+                self._entries(data, station), show_id,
+                txt_or_none(data.get('title')),
+                clean_html(data.get('subtitle'))))
+
+
+class ORFRadioCollectionIE(ORFRadioBase):
+    IE_NAME = 'orf:collection'
+    _VALID_URL = r'https?://sound\.orf\.at/collection/(?P<coll_id>\d+)(?:/(?P<item_id>\d+))?'
+
+    _TESTS = [{
+        'url': 'https://sound.orf.at/collection/4/61908/was-das-uberschreiten-des-15-limits-bedeutet',
+        'info_dict': {
+            'id': '2577582',
+        },
+        'playlist': [{
+            'md5': '5789cec7d75575ff58d19c0428c80eb3',
+            'info_dict': {
+                'id': '2024-06-06_1659_tl_54_7DaysThu6_153926',
+                'ext': 'mp3',
+                'title': 'Klimakrise: Was das Überschreiten des 1,5°-Limits bedeutet',
+                'timestamp': 1717686674,
+                'upload_date': '20240606',
+                'uploader': 'fm4',
+            },
+        }],
+        # 'skip': 'Shows from ORF Sound are only available for 30 days.'
+    }, {
+        'url': 'https://sound.orf.at/collection/4/',
+        'info_dict': {
+            'id': '4',
+        },
+        'playlist_mincount': 10,
+        'playlist_maxcount': 13,
+    }]
+
+    def _real_extract(self, url):
+        coll_id, item_id = self._match_valid_url(url).group('coll_id', 'item_id')
+        data = self._download_json(
+            'https://collector.orf.at/api/frontend/collections/{0}?_o=sound.orf.at'.format(
+                coll_id), coll_id)
+        data = self._get_api_payload(data, coll_id, in_payload=True)
+
+        def yield_items():
+            for item in traverse_obj(data, (
+                    'content', 'items', lambda _, v: any(k in v['target']['params'] for k in self._ID_NAMES))):
+                if item_id is None or item_id == txt_or_none(item.get('id')):
+                    target = item['target']
+                    typed_item_id = self._get_item_id(target['params'])
+                    station = target['params'].get('station')
+                    item_type = target.get('type')
+                    if typed_item_id and (station or item_type):
+                        yield station, typed_item_id, item_type
+                    if item_id is not None:
+                        break
+            else:
+                if item_id is not None:
+                    raise ExtractorError('Item not found in collection',
+                                         video_id=coll_id, expected=True)
+
+        def item_playlist(station, typed_item_id, item_type):
+            if item_type == 'upload':
+                item_data = self._download_json('https://audioapi.orf.at/radiothek/api/2.0/upload/{0}?_o=sound.orf.at'.format(
+                    typed_item_id), typed_item_id)
+            elif item_type == 'podcast-episode':
+                item_data = self._download_json('https://audioapi.orf.at/radiothek/api/2.0/episode/{0}?_o=sound.orf.at'.format(
+                    typed_item_id), typed_item_id)
+            else:
+                api_station, _, _ = self.STATION_INFO[station]
+                item_data = self._download_json(
+                    'https://audioapi.orf.at/{0}/api/json/5.0/{1}/{2}?_o=sound.orf.at'.format(
+                        api_station, item_type or 'broadcastitem', typed_item_id), typed_item_id)
+
+            item_data = self._get_api_payload(item_data, typed_item_id, in_payload=True)
+
+            return merge_dicts(
+                {'_type': 'multi_video'},
+                self.playlist_result(
+                    self._entries(item_data, station, item_type), typed_item_id,
+                    txt_or_none(data.get('title')),
+                    clean_html(data.get('subtitle'))))
+
+        def yield_item_entries():
+            for station, typed_id, item_type in yield_items():
+                yield item_playlist(station, typed_id, item_type)
+
+        if item_id is not None:
+            # coll_id = '/'.join((coll_id, item_id))
+            return next(yield_item_entries())
+
+        return self.playlist_result(yield_item_entries(), coll_id, data.get('title'))
+
+
+class ORFPodcastIE(ORFRadioBase):
+    IE_NAME = 'orf:podcast'
+    _STATION_RE = '|'.join(map(re.escape, (x[0] for x in ORFRadioBase.STATION_INFO.values()))) + '|tv'
+    _VALID_URL = r'https?://sound\.orf\.at/podcast/(?P<station>{0})/(?P<show>[\w-]+)/(?P<id>[\w-]+)'.format(_STATION_RE)
+    _TESTS = [{
+        'url': 'https://sound.orf.at/podcast/stm/der-kraeutertipp-von-christine-lackner/rotklee',
+        'md5': '1f2bab2ba90c2ce0c2754196ea78b35f',
+        'info_dict': {
+            'id': 'der-kraeutertipp-von-christine-lackner/rotklee',
+            'ext': 'mp3',
+            'title': 'Rotklee',
+            'description': 'In der Natur weit verbreitet - in der Medizin längst anerkennt: Rotklee. Dieser Podcast begleitet die Sendung "Radio Steiermark am Vormittag", Radio Steiermark, 28. Mai 2024.',
+            'timestamp': 1716891761,
+            'upload_date': '20240528',
+            'uploader_id': 'stm_kraeutertipp',
+            'uploader': 'ORF Radio Steiermark',
+            'duration': 101,
+            'series': 'Der Kräutertipp von Christine Lackner',
+        },
+        # 'skip': 'ORF podcasts are only available for a limited time'
+    }]
+
+    _ID_NAMES = ('slug', 'guid')
+
+    def _real_extract(self, url):
+        station, show, show_id = self._match_valid_url(url).group('station', 'show', 'id')
+        data = self._download_json(
+            'https://audioapi.orf.at/radiothek/api/2.0/podcast/{0}/{1}/{2}'.format(
+                station, show, show_id), show_id)
+        data = self._get_api_payload(data, show_id, in_payload=True)
+
+        return merge_dicts({
+            'id': '/'.join((show, show_id)),
+            'ext': 'mp3',
+            'vcodec': 'none',
+        }, self._extract_podcast_upload(data), rev=True)
 
 
 class ORFIPTVIE(InfoExtractor):

From e20ca543f037bd3a8e38507b870ed3a3de3c32e7 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 1 Jun 2024 13:43:37 +0100
Subject: [PATCH 764/860] [ORF] Re-factor and update`ORFFM4StoryIE` * fix
 getting media via DASH instead of inaccessible mp4 * also get in-page YT
 media

---
 youtube_dl/extractor/orf.py | 253 ++++++++++++++++++------------------
 1 file changed, 126 insertions(+), 127 deletions(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index 25c16c84d..f03aa40dc 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -6,6 +6,7 @@ import functools
 import re
 
 from .common import InfoExtractor
+from .youtube import YoutubeIE
 from ..utils import (
     clean_html,
     determine_ext,
@@ -14,10 +15,8 @@ from ..utils import (
     int_or_none,
     merge_dicts,
     mimetype2ext,
-    orderedSet,
     parse_age_limit,
     parse_iso8601,
-    remove_end,
     strip_jsonp,
     txt_or_none,
     unified_strdate,
@@ -305,11 +304,90 @@ class ORFPodcastIE(ORFRadioBase):
         }, self._extract_podcast_upload(data), rev=True)
 
 
-class ORFIPTVIE(InfoExtractor):
+class ORFIPTVBase(InfoExtractor):
+    _TITLE_STRIP_RE = ''
+
+    def _extract_video(self, video_id, webpage, fatal=False):
+
+        data = self._download_json(
+            'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
+            video_id)[0]
+
+        video = traverse_obj(data, (
+            'sources', ('default', 'q8c'),
+            T(lambda x: x if x['loadBalancerUrl'] else None),
+            any))
+
+        load_balancer_url = video['loadBalancerUrl']
+
+        try:
+            rendition = self._download_json(
+                load_balancer_url, video_id, transform_source=strip_jsonp)
+        except ExtractorError:
+            rendition = None
+
+        if not rendition:
+            rendition = {
+                'redirect': {
+                    'smil': re.sub(
+                        r'(/)jsonp(/.+\.)mp4$', r'\1dash\2smil/manifest.mpd',
+                        load_balancer_url),
+                },
+            }
+
+        f = traverse_obj(video, {
+            'abr': ('audioBitrate', T(int_or_none)),
+            'vbr': ('bitrate', T(int_or_none)),
+            'fps': ('videoFps', T(int_or_none)),
+            'width': ('videoWidth', T(int_or_none)),
+            'height': ('videoHeight', T(int_or_none)),
+        })
+
+        formats = []
+        for format_id, format_url in traverse_obj(rendition, (
+                'redirect', T(dict.items), Ellipsis)):
+            if format_id == 'rtmp':
+                ff = f.copy()
+                ff.update({
+                    'url': format_url,
+                    'format_id': format_id,
+                })
+                formats.append(ff)
+            elif determine_ext(format_url) == 'f4m':
+                formats.extend(self._extract_f4m_formats(
+                    format_url, video_id, f4m_id=format_id))
+            elif determine_ext(format_url) == 'm3u8':
+                formats.extend(self._extract_m3u8_formats(
+                    format_url, video_id, 'mp4', m3u8_id=format_id,
+                    entry_protocol='m3u8_native'))
+            elif determine_ext(format_url) == 'mpd':
+                formats.extend(self._extract_mpd_formats(
+                    format_url, video_id, mpd_id=format_id))
+
+        if formats or fatal:
+            self._sort_formats(formats)
+        else:
+            return
+
+        return merge_dicts({
+            'id': video_id,
+            'title': re.sub(self._TITLE_STRIP_RE, '', self._og_search_title(webpage)),
+            'description': self._og_search_description(webpage),
+            'upload_date': unified_strdate(self._html_search_meta(
+                'dc.date', webpage, 'upload date', fatal=False)),
+            'formats': formats,
+        }, traverse_obj(data, {
+            'duration': ('duration', T(k_float_or_none)),
+            'thumbnail': ('sources', 'default', 'preview', T(url_or_none)),
+        }), rev=True)
+
+
+class ORFIPTVIE(ORFIPTVBase):
     IE_NAME = 'orf:iptv'
     IE_DESC = 'iptv.ORF.at'
     _WORKING = False  # URLs redirect to orf.at/
     _VALID_URL = r'https?://iptv\.orf\.at/(?:#/)?stories/(?P<id>\d+)'
+    _TITLE_STRIP_RE = r'\s+-\s+iptv\.ORF\.at\S*$'
 
     _TEST = {
         'url': 'http://iptv.orf.at/stories/2275236/',
@@ -334,74 +412,32 @@ class ORFIPTVIE(InfoExtractor):
         video_id = self._search_regex(
             r'data-video(?:id)?="(\d+)"', webpage, 'video id')
 
-        data = self._download_json(
-            'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
-            video_id)[0]
-
-        duration = float_or_none(data['duration'], 1000)
-
-        video = data['sources']['default']
-        load_balancer_url = video['loadBalancerUrl']
-        abr = int_or_none(video.get('audioBitrate'))
-        vbr = int_or_none(video.get('bitrate'))
-        fps = int_or_none(video.get('videoFps'))
-        width = int_or_none(video.get('videoWidth'))
-        height = int_or_none(video.get('videoHeight'))
-        thumbnail = video.get('preview')
-
-        rendition = self._download_json(
-            load_balancer_url, video_id, transform_source=strip_jsonp)
-
-        f = {
-            'abr': abr,
-            'vbr': vbr,
-            'fps': fps,
-            'width': width,
-            'height': height,
-        }
-
-        formats = []
-        for format_id, format_url in rendition['redirect'].items():
-            if format_id == 'rtmp':
-                ff = f.copy()
-                ff.update({
-                    'url': format_url,
-                    'format_id': format_id,
-                })
-                formats.append(ff)
-            elif determine_ext(format_url) == 'f4m':
-                formats.extend(self._extract_f4m_formats(
-                    format_url, video_id, f4m_id=format_id))
-            elif determine_ext(format_url) == 'm3u8':
-                formats.extend(self._extract_m3u8_formats(
-                    format_url, video_id, 'mp4', m3u8_id=format_id))
-            else:
-                continue
-        self._sort_formats(formats)
-
-        title = remove_end(self._og_search_title(webpage), ' - iptv.ORF.at')
-        description = self._og_search_description(webpage)
-        upload_date = unified_strdate(self._html_search_meta(
-            'dc.date', webpage, 'upload date'))
-
-        return {
-            'id': video_id,
-            'title': title,
-            'description': description,
-            'duration': duration,
-            'thumbnail': thumbnail,
-            'upload_date': upload_date,
-            'formats': formats,
-        }
+        return self._extract_video(video_id, webpage)
 
 
-class ORFFM4StoryIE(InfoExtractor):
+class ORFFM4StoryIE(ORFIPTVBase):
     IE_NAME = 'orf:fm4:story'
     IE_DESC = 'fm4.orf.at stories'
     _VALID_URL = r'https?://fm4\.orf\.at/stories/(?P<id>\d+)'
+    _TITLE_STRIP_RE = r'\s+-\s+fm4\.ORF\.at\s*$'
 
-    _TEST = {
+    _TESTS = [{
+        'url': 'https://fm4.orf.at/stories/3041554/',
+        'add_ie': ['Youtube'],
+        'info_dict': {
+            'id': '3041554',
+            'title': 'Is The EU Green Deal In Mortal Danger?',
+        },
+        'playlist_count': 4,
+        'params': {
+            'format': 'bestvideo',
+        },
+    }, {
         'url': 'http://fm4.orf.at/stories/2865738/',
+        'info_dict': {
+            'id': '2865738',
+            'title': 'Manu Delago und Inner Tongue live',
+        },
         'playlist': [{
             'md5': 'e1c2c706c45c7b34cf478bbf409907ca',
             'info_dict': {
@@ -418,86 +454,49 @@ class ORFFM4StoryIE(InfoExtractor):
             'info_dict': {
                 'id': '547798',
                 'ext': 'flv',
-                'title': 'Manu Delago und Inner Tongue live (2)',
+                'title': 'Manu Delago und Inner Tongue https://vod-ww.mdn.ors.at/cms-worldwide_episodes_nas/_definst_/nas/cms-worldwide_episodes/online/14228823_0005.smil/chunklist_b992000_vo.m3u8live (2)',
                 'duration': 1504.08,
                 'thumbnail': r're:^https?://.*\.jpg$',
                 'upload_date': '20170913',
                 'description': 'Manu Delago und Inner Tongue haben bei der FM4 Soundpark Session live alles gegeben. Hier gibt es Fotos und die gesamte Session als Video.',
             },
         }],
-    }
+        'skip': 'Videos gone',
+    }]
 
     def _real_extract(self, url):
         story_id = self._match_id(url)
         webpage = self._download_webpage(url, story_id)
 
         entries = []
-        all_ids = orderedSet(re.findall(r'data-video(?:id)?="(\d+)"', webpage))
-        for idx, video_id in enumerate(all_ids):
-            data = self._download_json(
-                'http://bits.orf.at/filehandler/static-api/json/current/data.json?file=%s' % video_id,
-                video_id)[0]
+        seen_ids = set()
+        for idx, video_id in enumerate(re.findall(r'data-video(?:id)?="(\d+)"', webpage)):
+            if video_id in seen_ids:
+                continue
+            seen_ids.add(video_id)
+            entry = self._extract_video(video_id, webpage, fatal=False)
+            if not entry:
+                continue
 
-            duration = float_or_none(data['duration'], 1000)
-
-            video = data['sources']['q8c']
-            load_balancer_url = video['loadBalancerUrl']
-            abr = int_or_none(video.get('audioBitrate'))
-            vbr = int_or_none(video.get('bitrate'))
-            fps = int_or_none(video.get('videoFps'))
-            width = int_or_none(video.get('videoWidth'))
-            height = int_or_none(video.get('videoHeight'))
-            thumbnail = video.get('preview')
-
-            rendition = self._download_json(
-                load_balancer_url, video_id, transform_source=strip_jsonp)
-
-            f = {
-                'abr': abr,
-                'vbr': vbr,
-                'fps': fps,
-                'width': width,
-                'height': height,
-            }
-
-            formats = []
-            for format_id, format_url in rendition['redirect'].items():
-                if format_id == 'rtmp':
-                    ff = f.copy()
-                    ff.update({
-                        'url': format_url,
-                        'format_id': format_id,
-                    })
-                    formats.append(ff)
-                elif determine_ext(format_url) == 'f4m':
-                    formats.extend(self._extract_f4m_formats(
-                        format_url, video_id, f4m_id=format_id))
-                elif determine_ext(format_url) == 'm3u8':
-                    formats.extend(self._extract_m3u8_formats(
-                        format_url, video_id, 'mp4', m3u8_id=format_id))
-                else:
-                    continue
-            self._sort_formats(formats)
-
-            title = remove_end(self._og_search_title(webpage), ' - fm4.ORF.at')
             if idx >= 1:
                 # Titles are duplicates, make them unique
-                title += ' (' + str(idx + 1) + ')'
-            description = self._og_search_description(webpage)
-            upload_date = unified_strdate(self._html_search_meta(
-                'dc.date', webpage, 'upload date'))
+                entry['title'] = '%s (%d)' % (entry['title'], idx)
 
-            entries.append({
-                'id': video_id,
-                'title': title,
-                'description': description,
-                'duration': duration,
-                'thumbnail': thumbnail,
-                'upload_date': upload_date,
-                'formats': formats,
-            })
+            entries.append(entry)
 
-        return self.playlist_result(entries)
+        seen_ids = set()
+        for yt_id in re.findall(
+                r'data-id\s*=\s*["\']([\w-]+)[^>]+\bclass\s*=\s*["\']youtube\b',
+                webpage):
+            if yt_id in seen_ids:
+                continue
+            seen_ids.add(yt_id)
+            if YoutubeIE.suitable(yt_id):
+                entries.append(self.url_result(yt_id, ie='Youtube', video_id=yt_id))
+
+        return self.playlist_result(
+            entries, story_id,
+            re.sub(self._TITLE_STRIP_RE, '', self._og_search_title(webpage, default='') or None))
 
 
 class ORFONBase(InfoExtractor):

From a48fe7491df1ef9c1d10689e852f0e8e5f48253c Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 11 Jun 2024 12:32:07 +0100
Subject: [PATCH 765/860] [ORF] Skip tests with limited availability

---
 youtube_dl/extractor/orf.py | 11 +++++++----
 1 file changed, 7 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/orf.py b/youtube_dl/extractor/orf.py
index f03aa40dc..1ee78edbc 100644
--- a/youtube_dl/extractor/orf.py
+++ b/youtube_dl/extractor/orf.py
@@ -135,7 +135,7 @@ class ORFRadioIE(ORFRadioBase):
                 'duration': 14413.0,
             }
         }],
-        # 'skip': 'Shows from ORF Sound are only available for 30 days.'
+        'skip': 'Shows from ORF Sound are only available for 30 days.'
     }, {
         'url': 'https://oe1.orf.at/player/20240531/758136',
         'md5': '2397717aaf3ae9c22a4f090ee3b8d374',
@@ -149,7 +149,7 @@ class ORFRadioIE(ORFRadioBase):
             'uploader': 'oe1',
             'duration': 1500,
         },
-        # 'skip': 'Shows from ORF Sound are only available for 30 days.'
+        'skip': 'Shows from ORF Sound are only available for 30 days.'
     }]
 
     def _real_extract(self, url):
@@ -199,8 +199,9 @@ class ORFRadioCollectionIE(ORFRadioBase):
                 'uploader': 'fm4',
             },
         }],
-        # 'skip': 'Shows from ORF Sound are only available for 30 days.'
+        'skip': 'Shows from ORF Sound are only available for 30 days.'
     }, {
+        # persistent playlist (FM4 Highlights)
         'url': 'https://sound.orf.at/collection/4/',
         'info_dict': {
             'id': '4',
@@ -285,7 +286,7 @@ class ORFPodcastIE(ORFRadioBase):
             'duration': 101,
             'series': 'Der Kräutertipp von Christine Lackner',
         },
-        # 'skip': 'ORF podcasts are only available for a limited time'
+        'skip': 'ORF podcasts are only available for a limited time'
     }]
 
     _ID_NAMES = ('slug', 'guid')
@@ -642,6 +643,7 @@ class ORFONIE(ORFONBase):
         'params': {
             'format': 'bestvideo',
         },
+        'skip': 'Available until 2024-08-12',
     }, {
         'url': 'https://on.orf.at/video/3220355',
         'md5': '925a93b2b9a37da5c9b979d7cf71aa2e',
@@ -683,6 +685,7 @@ class ORFONIE(ORFONBase):
             'noplaylist': True,
             'format': 'bestvideo',
         },
+        'skip': 'Available until 2024-06-14',
     }, {
         # Video with multiple segments and no combined version
         'url': 'https://on.orf.at/video/14227864/formel-1-grosser-preis-von-monaco-2024',

From 0153b387e57e0bb8e580f1869f85596d2767fb0d Mon Sep 17 00:00:00 2001
From: Paper <paper@paper.us.eu.org>
Date: Tue, 11 Jun 2024 08:21:39 -0400
Subject: [PATCH 766/860] [VidLii] Add 720p support (#30924)

* [VidLii] Add HD support  (yt-dlp backport-ish)

* Also fix a bug with the view count

---------

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 youtube_dl/extractor/vidlii.py | 59 +++++++++++++++++++++++++++++-----
 1 file changed, 51 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/vidlii.py b/youtube_dl/extractor/vidlii.py
index f4774256b..47f328e87 100644
--- a/youtube_dl/extractor/vidlii.py
+++ b/youtube_dl/extractor/vidlii.py
@@ -4,6 +4,7 @@ from __future__ import unicode_literals
 import re
 
 from .common import InfoExtractor
+
 from ..utils import (
     float_or_none,
     get_element_by_id,
@@ -11,6 +12,7 @@ from ..utils import (
     strip_or_none,
     unified_strdate,
     urljoin,
+    str_to_int,
 )
 
 
@@ -35,6 +37,26 @@ class VidLiiIE(InfoExtractor):
             'categories': ['News & Politics'],
             'tags': ['Vidlii', 'Jan', 'Videogames'],
         }
+    }, {
+        # HD
+        'url': 'https://www.vidlii.com/watch?v=2Ng8Abj2Fkl',
+        'md5': '450e7da379c884788c3a4fa02a3ce1a4',
+        'info_dict': {
+            'id': '2Ng8Abj2Fkl',
+            'ext': 'mp4',
+            'title': 'test',
+            'description': 'md5:cc55a86032a7b6b3cbfd0f6b155b52e9',
+            'thumbnail': 'https://www.vidlii.com/usfi/thmp/2Ng8Abj2Fkl.jpg',
+            'uploader': 'VidLii',
+            'uploader_url': 'https://www.vidlii.com/user/VidLii',
+            'upload_date': '20200927',
+            'duration': 5,
+            'view_count': int,
+            'comment_count': int,
+            'average_rating': float,
+            'categories': ['Film & Animation'],
+            'tags': list,
+        },
     }, {
         'url': 'https://www.vidlii.com/embed?v=tJluaH4BJ3v&a=0',
         'only_matching': True,
@@ -46,11 +68,32 @@ class VidLiiIE(InfoExtractor):
         webpage = self._download_webpage(
             'https://www.vidlii.com/watch?v=%s' % video_id, video_id)
 
-        video_url = self._search_regex(
-            r'src\s*:\s*(["\'])(?P<url>(?:https?://)?(?:(?!\1).)+)\1', webpage,
-            'video url', group='url')
+        formats = []
 
-        title = self._search_regex(
+        def add_format(format_url, height=None):
+            height = int(self._search_regex(r'(\d+)\.mp4',
+                         format_url, 'height', default=360))
+
+            formats.append({
+                'url': format_url,
+                'format_id': '%dp' % height if height else None,
+                'height': height,
+            })
+
+        sources = re.findall(
+            r'src\s*:\s*(["\'])(?P<url>(?:https?://)?(?:(?!\1).)+)\1',
+            webpage)
+
+        formats = []
+        if len(sources) > 1:
+            add_format(sources[1][1])
+            self._check_formats(formats, video_id)
+        if len(sources) > 0:
+            add_format(sources[0][1])
+
+        self._sort_formats(formats)
+
+        title = self._html_search_regex(
             (r'<h1>([^<]+)</h1>', r'<title>([^<]+) - VidLii<'), webpage,
             'title')
 
@@ -82,9 +125,9 @@ class VidLiiIE(InfoExtractor):
             default=None) or self._search_regex(
             r'duration\s*:\s*(\d+)', webpage, 'duration', fatal=False))
 
-        view_count = int_or_none(self._search_regex(
-            (r'<strong>(\d+)</strong> views',
-             r'Views\s*:\s*<strong>(\d+)</strong>'),
+        view_count = str_to_int(self._html_search_regex(
+            (r'<strong>([\d,.]+)</strong> views',
+             r'Views\s*:\s*<strong>([\d,.]+)</strong>'),
             webpage, 'view count', fatal=False))
 
         comment_count = int_or_none(self._search_regex(
@@ -109,7 +152,7 @@ class VidLiiIE(InfoExtractor):
 
         return {
             'id': video_id,
-            'url': video_url,
+            'formats': formats,
             'title': title,
             'description': description,
             'thumbnail': thumbnail,

From 2eac0fa3799b3d027148341186a52fb5a6288473 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 19 Jun 2024 19:04:47 +0100
Subject: [PATCH 767/860] [utils] Save `orig_msg` in `ExtractorError`

---
 youtube_dl/utils.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index cd4303566..113c913df 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -2406,7 +2406,7 @@ class ExtractorError(YoutubeDLError):
         """ tb, if given, is the original traceback (so that it can be printed out).
         If expected is set, this is a normal error message and most likely not a bug in youtube-dl.
         """
-
+        self.orig_msg = msg
         if sys.exc_info()[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError):
             expected = True
         if video_id is not None:

From ad01fa6ccadd1ecade8002e937492a141d3b8f25 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 19 Jun 2024 21:44:27 +0100
Subject: [PATCH 768/860] [jsinterp] Add Debugger from yt-dlp *
 https://github.com/yt-dlp/yt-dlp/commit/8f53dc4 * thx pukkandan

---
 test/test_jsinterp.py          |  8 ++++---
 test/test_youtube_signature.py |  4 ++++
 youtube_dl/extractor/common.py |  1 -
 youtube_dl/jsinterp.py         | 42 +++++++++++++++++++++++++++++++++-
 4 files changed, 50 insertions(+), 5 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 91b12f544..da8e98020 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -577,9 +577,11 @@ class TestJSInterpreter(unittest.TestCase):
     def test_unary_operators(self):
         jsi = JSInterpreter('function f(){return 2  -  - - 2;}')
         self.assertEqual(jsi.call_function('f'), 0)
-        # fails
-        # jsi = JSInterpreter('function f(){return 2 + - + - - 2;}')
-        # self.assertEqual(jsi.call_function('f'), 0)
+        jsi = JSInterpreter('function f(){return 2 + - + - - 2;}')
+        self.assertEqual(jsi.call_function('f'), 0)
+        # https://github.com/ytdl-org/youtube-dl/issues/32815
+        jsi = JSInterpreter('function f(){return 0  - 7 * - 6;}')
+        self.assertEqual(jsi.call_function('f'), 42)
 
     """ # fails so far
     def test_packed(self):
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index f45dfec7c..cafba7a5c 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -158,6 +158,10 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/b7910ca8/player_ias.vflset/en_US/base.js',
         '_hXMCwMt9qE310D', 'LoZMgkkofRMCZQ',
     ),
+    (
+        'https://www.youtube.com/s/player/590f65a6/player_ias.vflset/en_US/base.js',
+        '1tm7-g_A9zsI8_Lay_', 'xI4Vem4Put_rOg',
+    ),
 ]
 
 
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index b10e84416..9b0016d07 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -3033,7 +3033,6 @@ class InfoExtractor(object):
             transform_source=transform_source, default=None)
 
     def _extract_jwplayer_data(self, webpage, video_id, *args, **kwargs):
-
         # allow passing `transform_source` through to _find_jwplayer_data()
         transform_source = kwargs.pop('transform_source', None)
         kwfind = compat_kwargs({'transform_source': transform_source}) if transform_source else {}
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 86d902248..e258ebd00 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -14,6 +14,7 @@ from .utils import (
     remove_quotes,
     unified_timestamp,
     variadic,
+    write_string,
 )
 from .compat import (
     compat_basestring,
@@ -220,6 +221,42 @@ class LocalNameSpace(ChainMap):
         return 'LocalNameSpace%s' % (self.maps, )
 
 
+class Debugger(object):
+    ENABLED = False
+
+    @staticmethod
+    def write(*args, **kwargs):
+        level = kwargs.get('level', 100)
+
+        def truncate_string(s, left, right=0):
+            if s is None or len(s) <= left + right:
+                return s
+            return '...'.join((s[:left - 3], s[-right:] if right else ''))
+
+        write_string('[debug] JS: {0}{1}\n'.format(
+            '  ' * (100 - level),
+            ' '.join(truncate_string(compat_str(x), 50, 50) for x in args)))
+
+    @classmethod
+    def wrap_interpreter(cls, f):
+        def interpret_statement(self, stmt, local_vars, allow_recursion, *args, **kwargs):
+            if cls.ENABLED and stmt.strip():
+                cls.write(stmt, level=allow_recursion)
+            try:
+                ret, should_ret = f(self, stmt, local_vars, allow_recursion, *args, **kwargs)
+            except Exception as e:
+                if cls.ENABLED:
+                    if isinstance(e, ExtractorError):
+                        e = e.orig_msg
+                    cls.write('=> Raises:', e, '<-|', stmt, level=allow_recursion)
+                raise
+            if cls.ENABLED and stmt.strip():
+                if should_ret or not repr(ret) == stmt:
+                    cls.write(['->', '=>'][should_ret], repr(ret), '<-|', stmt, level=allow_recursion)
+            return ret, should_ret
+        return interpret_statement
+
+
 class JSInterpreter(object):
     __named_object_counter = 0
 
@@ -416,7 +453,7 @@ class JSInterpreter(object):
         except Exception as e:
             if allow_undefined:
                 return JS_Undefined
-            raise self.Exception('Cannot get index {idx:.100}'.format(**locals()), expr=repr(obj), cause=e)
+            raise self.Exception('Cannot get index {idx!r:.100}'.format(**locals()), expr=repr(obj), cause=e)
 
     def _dump(self, obj, namespace):
         try:
@@ -438,6 +475,7 @@ class JSInterpreter(object):
     _FINALLY_RE = re.compile(r'finally\s*\{')
     _SWITCH_RE = re.compile(r'switch\s*\(')
 
+    @Debugger.wrap_interpreter
     def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if allow_recursion < 0:
             raise self.Exception('Recursion limit reached')
@@ -797,6 +835,8 @@ class JSInterpreter(object):
 
             def eval_method():
                 if (variable, member) == ('console', 'debug'):
+                    if Debugger.ENABLED:
+                        Debugger.write(self.interpret_expression('[{}]'.format(arg_str), local_vars, allow_recursion))
                     return
                 types = {
                     'String': compat_str,

From fd8242e3efd3c0e2ba9a45c662d6983c00b21d6d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 20 Jun 2024 02:36:54 +0100
Subject: [PATCH 769/860] [jsinterp] Fix and improve expression parsing *
 improve BODMAS (fixes https://github.com/ytdl-org/youtube-dl/issues/32815) *
 support more weird expressions with multiple unary ops

---
 youtube_dl/jsinterp.py | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index e258ebd00..12b71ed6a 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -798,18 +798,28 @@ class JSInterpreter(object):
             right_expr = separated.pop()
             # handle operators that are both unary and binary, minimal BODMAS
             if op in ('+', '-'):
+                # simplify/adjust consecutive instances of these operators
                 undone = 0
                 while len(separated) > 1 and not separated[-1].strip():
                     undone += 1
                     separated.pop()
                 if op == '-' and undone % 2 != 0:
                     right_expr = op + right_expr
+                elif op == '+':
+                    while len(separated) > 1 and separated[-1].strip() in self.OP_CHARS:
+                        right_expr = separated.pop() + right_expr
+                # hanging op at end of left => unary + (strip) or - (push right)
                 left_val = separated[-1]
                 for dm_op in ('*', '%', '/', '**'):
                     bodmas = tuple(self._separate(left_val, dm_op, skip_delims=skip_delim))
                     if len(bodmas) > 1 and not bodmas[-1].strip():
                         expr = op.join(separated) + op + right_expr
-                        right_expr = None
+                        if len(separated) > 1:
+                            separated.pop()
+                            right_expr = op.join((left_val, right_expr))
+                        else:
+                            separated = [op.join((left_val, right_expr))]
+                            right_expr = None
                         break
                 if right_expr is None:
                     continue

From e0094e63c3bce919841b6e3634c877fcef8772a1 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 20 Jun 2024 02:57:52 +0100
Subject: [PATCH 770/860] [jsinterp] Various tweaks * treat Infinity like NaN *
 cache operator list

---
 youtube_dl/jsinterp.py | 31 ++++++++++++++++---------------
 1 file changed, 16 insertions(+), 15 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 12b71ed6a..02adf6678 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -54,15 +54,16 @@ def wraps_op(op):
 
 # NB In principle NaN cannot be checked by membership.
 # Here all NaN values are actually this one, so _NaN is _NaN,
-# although _NaN != _NaN.
+# although _NaN != _NaN. Ditto Infinity.
 
 _NaN = float('nan')
+_Infinity = float('inf')
 
 
 def _js_bit_op(op):
 
     def zeroise(x):
-        return 0 if x in (None, JS_Undefined, _NaN) else x
+        return 0 if x in (None, JS_Undefined, _NaN, _Infinity) else x
 
     @wraps_op(op)
     def wrapped(a, b):
@@ -85,7 +86,7 @@ def _js_arith_op(op):
 def _js_div(a, b):
     if JS_Undefined in (a, b) or not (a or b):
         return _NaN
-    return operator.truediv(a or 0, b) if b else float('inf')
+    return operator.truediv(a or 0, b) if b else _Infinity
 
 
 def _js_mod(a, b):
@@ -344,8 +345,7 @@ class JSInterpreter(object):
     def __op_chars(cls):
         op_chars = set(';,[')
         for op in cls._all_operators():
-            for c in op[0]:
-                op_chars.add(c)
+            op_chars.update(op[0])
         return op_chars
 
     def _named_object(self, namespace, obj):
@@ -363,9 +363,8 @@ class JSInterpreter(object):
         # collections.Counter() is ~10% slower in both 2.7 and 3.9
         counters = dict((k, 0) for k in _MATCHING_PARENS.values())
         start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
-        in_quote, escaping, skipping = None, False, 0
-        after_op, in_regex_char_group = True, False
-
+        in_quote, escaping, after_op, in_regex_char_group = None, False, True, False
+        skipping = 0
         for idx, char in enumerate(expr):
             paren_delta = 0
             if not in_quote:
@@ -419,10 +418,12 @@ class JSInterpreter(object):
         return separated[0][1:].strip(), separated[1].strip()
 
     @staticmethod
-    def _all_operators():
-        return itertools.chain(
-            # Ref: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_Precedence
-            _SC_OPERATORS, _LOG_OPERATORS, _COMP_OPERATORS, _OPERATORS)
+    def _all_operators(_cached=[]):
+        if not _cached:
+            _cached.extend(itertools.chain(
+                # Ref: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_Precedence
+                _SC_OPERATORS, _LOG_OPERATORS, _COMP_OPERATORS, _OPERATORS))
+        return _cached
 
     def _operator(self, op, left_val, right_expr, expr, local_vars, allow_recursion):
         if op in ('||', '&&'):
@@ -549,7 +550,6 @@ class JSInterpreter(object):
                 expr = self._dump(inner, local_vars) + outer
 
         if expr.startswith('('):
-
             m = re.match(r'\((?P<d>[a-z])%(?P<e>[a-z])\.length\+(?P=e)\.length\)%(?P=e)\.length', expr)
             if m:
                 # short-cut eval of frequently used `(d%e.length+e.length)%e.length`, worth ~6% on `pytest -k test_nsig`
@@ -731,7 +731,7 @@ class JSInterpreter(object):
                 (?P<op>{_OPERATOR_RE})?
                 =(?!=)(?P<expr>.*)$
             )|(?P<return>
-                (?!if|return|true|false|null|undefined)(?P<name>{_NAME_RE})$
+                (?!if|return|true|false|null|undefined|NaN|Infinity)(?P<name>{_NAME_RE})$
             )|(?P<indexing>
                 (?P<in>{_NAME_RE})\[(?P<idx>.+)\]$
             )|(?P<attribute>
@@ -765,11 +765,12 @@ class JSInterpreter(object):
             raise JS_Break()
         elif expr == 'continue':
             raise JS_Continue()
-
         elif expr == 'undefined':
             return JS_Undefined, should_return
         elif expr == 'NaN':
             return _NaN, should_return
+        elif expr == 'Infinity':
+            return _Infinity, should_return
 
         elif md.get('return'):
             return local_vars[m.group('name')], should_return

From 4d05f84325070c3f6fe2ed6096138757675469a4 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 20 Jun 2024 03:22:02 +0100
Subject: [PATCH 771/860] [PalcoMP3] Conform to new linter rule * no space
 after @ in decorator

---
 youtube_dl/extractor/palcomp3.py | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/youtube_dl/extractor/palcomp3.py b/youtube_dl/extractor/palcomp3.py
index fb29d83f9..60f7a4d48 100644
--- a/youtube_dl/extractor/palcomp3.py
+++ b/youtube_dl/extractor/palcomp3.py
@@ -8,7 +8,7 @@ from ..compat import compat_str
 from ..utils import (
     int_or_none,
     str_or_none,
-    try_get,
+    traverse_obj,
 )
 
 
@@ -109,7 +109,7 @@ class PalcoMP3ArtistIE(PalcoMP3BaseIE):
     }
     name'''
 
-    @ classmethod
+    @classmethod
     def suitable(cls, url):
         return False if re.match(PalcoMP3IE._VALID_URL, url) else super(PalcoMP3ArtistIE, cls).suitable(url)
 
@@ -118,7 +118,8 @@ class PalcoMP3ArtistIE(PalcoMP3BaseIE):
         artist = self._call_api(artist_slug, self._ARTIST_FIELDS_TMPL)['artist']
 
         def entries():
-            for music in (try_get(artist, lambda x: x['musics']['nodes'], list) or []):
+            for music in traverse_obj(artist, (
+                    'musics', 'nodes', lambda _, m: m['musicID'])):
                 yield self._parse_music(music)
 
         return self.playlist_result(
@@ -137,7 +138,7 @@ class PalcoMP3VideoIE(PalcoMP3BaseIE):
             'title': 'Maiara e Maraisa - Você Faz Falta Aqui - DVD Ao Vivo Em Campo Grande',
             'description': 'md5:7043342c09a224598e93546e98e49282',
             'upload_date': '20161107',
-            'uploader_id': 'maiaramaraisaoficial',
+            'uploader_id': '@maiaramaraisaoficial',
             'uploader': 'Maiara e Maraisa',
         }
     }]

From 3c466186a86a22c3fc050f038c38b76ffa1a2dad Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 30 Jun 2024 12:34:19 +0100
Subject: [PATCH 772/860] [utils] Back-port Namespace and MEDIA_EXTENSIONS from
 yt-dlp Thx pukkandan * Namespace:
 https://github.com/yt-dlp/yt-dlp/commit/591bb9d355 * MEDIA_EXTENSIONS:
 https://github.com/yt-dlp/yt-dlp/commit/8dc5930511

---
 youtube_dl/utils.py | 44 +++++++++++++++++++++++++++++++-------------
 1 file changed, 31 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 113c913df..1af3e2b57 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1717,20 +1717,38 @@ TIMEZONE_NAMES = {
     'PST': -8, 'PDT': -7   # Pacific
 }
 
+
+class Namespace(object):
+    """Immutable namespace"""
+
+    def __init__(self, **kw_attr):
+        self.__dict__.update(kw_attr)
+
+    def __iter__(self):
+        return iter(self.__dict__.values())
+
+    @property
+    def items_(self):
+        return self.__dict__.items()
+
+
+MEDIA_EXTENSIONS = Namespace(
+    common_video=('avi', 'flv', 'mkv', 'mov', 'mp4', 'webm'),
+    video=('3g2', '3gp', 'f4v', 'mk3d', 'divx', 'mpg', 'ogv', 'm4v', 'wmv'),
+    common_audio=('aiff', 'alac', 'flac', 'm4a', 'mka', 'mp3', 'ogg', 'opus', 'wav'),
+    audio=('aac', 'ape', 'asf', 'f4a', 'f4b', 'm4b', 'm4p', 'm4r', 'oga', 'ogx', 'spx', 'vorbis', 'wma', 'weba'),
+    thumbnails=('jpg', 'png', 'webp'),
+    # storyboards=('mhtml', ),
+    subtitles=('srt', 'vtt', 'ass', 'lrc', 'ttml'),
+    manifests=('f4f', 'f4m', 'm3u8', 'smil', 'mpd'),
+)
+MEDIA_EXTENSIONS.video = MEDIA_EXTENSIONS.common_video + MEDIA_EXTENSIONS.video
+MEDIA_EXTENSIONS.audio = MEDIA_EXTENSIONS.common_audio + MEDIA_EXTENSIONS.audio
+
 KNOWN_EXTENSIONS = (
-    'mp4', 'm4a', 'm4p', 'm4b', 'm4r', 'm4v', 'aac',
-    'flv', 'f4v', 'f4a', 'f4b',
-    'webm', 'ogg', 'ogv', 'oga', 'ogx', 'spx', 'opus',
-    'mkv', 'mka', 'mk3d',
-    'avi', 'divx',
-    'mov',
-    'asf', 'wmv', 'wma',
-    '3gp', '3g2',
-    'mp3',
-    'flac',
-    'ape',
-    'wav',
-    'f4f', 'f4m', 'm3u8', 'smil')
+    MEDIA_EXTENSIONS.video + MEDIA_EXTENSIONS.audio
+    + MEDIA_EXTENSIONS.manifests
+)
 
 # needed for sanitizing filenames in restricted mode
 ACCENT_CHARS = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñòóôõöőøœùúûüűýþÿ',

From 46521096433aceaa41b4caa845bed22ca6f377ce Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 30 Jun 2024 18:37:25 +0100
Subject: [PATCH 773/860] [core,utils] Implement unsafe file extension
 mitigation * from
 https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-79w7-vh3h-8g4, thx
 grub4k

---
 test/test_utils.py      |  46 ++++++++++
 youtube_dl/YoutubeDL.py |  17 ++++
 youtube_dl/utils.py     | 189 +++++++++++++++++++++++++++++++---------
 3 files changed, 209 insertions(+), 43 deletions(-)

diff --git a/test/test_utils.py b/test/test_utils.py
index de7fe80b8..2947cce7e 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -14,9 +14,11 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 import io
 import itertools
 import json
+import types
 import xml.etree.ElementTree
 
 from youtube_dl.utils import (
+    _UnsafeExtensionError,
     age_restricted,
     args_to_str,
     base_url,
@@ -270,6 +272,27 @@ class TestUtil(unittest.TestCase):
             expand_path('~/%s' % env('YOUTUBE_DL_EXPATH_PATH')),
             '%s/expanded' % compat_getenv('HOME'))
 
+    _uncommon_extensions = [
+        ('exe', 'abc.exe.ext'),
+        ('de', 'abc.de.ext'),
+        ('../.mp4', None),
+        ('..\\.mp4', None),
+    ]
+
+    def assertUnsafeExtension(self, ext=None):
+        assert_raises = self.assertRaises(_UnsafeExtensionError)
+        assert_raises.ext = ext
+        orig_exit = assert_raises.__exit__
+
+        def my_exit(self_, exc_type, exc_val, exc_tb):
+            did_raise = orig_exit(exc_type, exc_val, exc_tb)
+            if did_raise and assert_raises.ext is not None:
+                self.assertEqual(assert_raises.ext, assert_raises.exception.extension, 'Unsafe extension  not as unexpected')
+            return did_raise
+
+        assert_raises.__exit__ = types.MethodType(my_exit, assert_raises)
+        return assert_raises
+
     def test_prepend_extension(self):
         self.assertEqual(prepend_extension('abc.ext', 'temp'), 'abc.temp.ext')
         self.assertEqual(prepend_extension('abc.ext', 'temp', 'ext'), 'abc.temp.ext')
@@ -278,6 +301,19 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(prepend_extension('.abc', 'temp'), '.abc.temp')
         self.assertEqual(prepend_extension('.abc.ext', 'temp'), '.abc.temp.ext')
 
+        # Test uncommon extensions
+        self.assertEqual(prepend_extension('abc.ext', 'bin'), 'abc.bin.ext')
+        for ext, result in self._uncommon_extensions:
+            with self.assertUnsafeExtension(ext):
+                prepend_extension('abc', ext)
+            if result:
+                self.assertEqual(prepend_extension('abc.ext', ext, 'ext'), result)
+            else:
+                with self.assertUnsafeExtension(ext):
+                    prepend_extension('abc.ext', ext, 'ext')
+            with self.assertUnsafeExtension(ext):
+                prepend_extension('abc.unexpected_ext', ext, 'ext')
+
     def test_replace_extension(self):
         self.assertEqual(replace_extension('abc.ext', 'temp'), 'abc.temp')
         self.assertEqual(replace_extension('abc.ext', 'temp', 'ext'), 'abc.temp')
@@ -286,6 +322,16 @@ class TestUtil(unittest.TestCase):
         self.assertEqual(replace_extension('.abc', 'temp'), '.abc.temp')
         self.assertEqual(replace_extension('.abc.ext', 'temp'), '.abc.temp')
 
+        # Test uncommon extensions
+        self.assertEqual(replace_extension('abc.ext', 'bin'), 'abc.unknown_video')
+        for ext, _ in self._uncommon_extensions:
+            with self.assertUnsafeExtension(ext):
+                replace_extension('abc', ext)
+            with self.assertUnsafeExtension(ext):
+                replace_extension('abc.ext', ext, 'ext')
+            with self.assertUnsafeExtension(ext):
+                replace_extension('abc.unexpected_ext', ext, 'ext')
+
     def test_subtitles_filename(self):
         self.assertEqual(subtitles_filename('abc.ext', 'en', 'vtt'), 'abc.en.vtt')
         self.assertEqual(subtitles_filename('abc.ext', 'en', 'vtt', 'ext'), 'abc.en.vtt')
diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index dad44435f..c19501915 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -7,6 +7,7 @@ import collections
 import copy
 import datetime
 import errno
+import functools
 import io
 import itertools
 import json
@@ -53,6 +54,7 @@ from .compat import (
     compat_urllib_request_DataHandler,
 )
 from .utils import (
+    _UnsafeExtensionError,
     age_restricted,
     args_to_str,
     bug_reports_message,
@@ -129,6 +131,20 @@ if compat_os_name == 'nt':
     import ctypes
 
 
+def _catch_unsafe_file_extension(func):
+    @functools.wraps(func)
+    def wrapper(self, *args, **kwargs):
+        try:
+            return func(self, *args, **kwargs)
+        except _UnsafeExtensionError as error:
+            self.report_error(
+                '{0} found; to avoid damaging your system, this value is disallowed.'
+                ' If you believe this is an error{1}').format(
+                    error.message, bug_reports_message(','))
+
+    return wrapper
+
+
 class YoutubeDL(object):
     """YoutubeDL class.
 
@@ -1925,6 +1941,7 @@ class YoutubeDL(object):
         if self.params.get('forcejson', False):
             self.to_stdout(json.dumps(self.sanitize_info(info_dict)))
 
+    @_catch_unsafe_file_extension
     def process_info(self, info_dict):
         """Process a single resolved IE result."""
 
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 1af3e2b57..df203b97a 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -1717,39 +1717,6 @@ TIMEZONE_NAMES = {
     'PST': -8, 'PDT': -7   # Pacific
 }
 
-
-class Namespace(object):
-    """Immutable namespace"""
-
-    def __init__(self, **kw_attr):
-        self.__dict__.update(kw_attr)
-
-    def __iter__(self):
-        return iter(self.__dict__.values())
-
-    @property
-    def items_(self):
-        return self.__dict__.items()
-
-
-MEDIA_EXTENSIONS = Namespace(
-    common_video=('avi', 'flv', 'mkv', 'mov', 'mp4', 'webm'),
-    video=('3g2', '3gp', 'f4v', 'mk3d', 'divx', 'mpg', 'ogv', 'm4v', 'wmv'),
-    common_audio=('aiff', 'alac', 'flac', 'm4a', 'mka', 'mp3', 'ogg', 'opus', 'wav'),
-    audio=('aac', 'ape', 'asf', 'f4a', 'f4b', 'm4b', 'm4p', 'm4r', 'oga', 'ogx', 'spx', 'vorbis', 'wma', 'weba'),
-    thumbnails=('jpg', 'png', 'webp'),
-    # storyboards=('mhtml', ),
-    subtitles=('srt', 'vtt', 'ass', 'lrc', 'ttml'),
-    manifests=('f4f', 'f4m', 'm3u8', 'smil', 'mpd'),
-)
-MEDIA_EXTENSIONS.video = MEDIA_EXTENSIONS.common_video + MEDIA_EXTENSIONS.video
-MEDIA_EXTENSIONS.audio = MEDIA_EXTENSIONS.common_audio + MEDIA_EXTENSIONS.audio
-
-KNOWN_EXTENSIONS = (
-    MEDIA_EXTENSIONS.video + MEDIA_EXTENSIONS.audio
-    + MEDIA_EXTENSIONS.manifests
-)
-
 # needed for sanitizing filenames in restricted mode
 ACCENT_CHARS = dict(zip('ÂÃÄÀÁÅÆÇÈÉÊËÌÍÎÏÐÑÒÓÔÕÖŐØŒÙÚÛÜŰÝÞßàáâãäåæçèéêëìíîïðñòóôõöőøœùúûüűýþÿ',
                         itertools.chain('AAAAAA', ['AE'], 'CEEEEIIIIDNOOOOOOO', ['OE'], 'UUUUUY', ['TH', 'ss'],
@@ -3977,19 +3944,22 @@ def parse_duration(s):
     return duration
 
 
-def prepend_extension(filename, ext, expected_real_ext=None):
+def _change_extension(prepend, filename, ext, expected_real_ext=None):
     name, real_ext = os.path.splitext(filename)
-    return (
-        '{0}.{1}{2}'.format(name, ext, real_ext)
-        if not expected_real_ext or real_ext[1:] == expected_real_ext
-        else '{0}.{1}'.format(filename, ext))
+    sanitize_extension = _UnsafeExtensionError.sanitize_extension
+
+    if not expected_real_ext or real_ext.partition('.')[0::2] == ('', expected_real_ext):
+        filename = name
+        if prepend and real_ext:
+            sanitize_extension(ext, prepend=prepend)
+            return ''.join((filename, '.', ext, real_ext))
+
+    # Mitigate path traversal and file impersonation attacks
+    return '.'.join((filename, sanitize_extension(ext)))
 
 
-def replace_extension(filename, ext, expected_real_ext=None):
-    name, real_ext = os.path.splitext(filename)
-    return '{0}.{1}'.format(
-        name if not expected_real_ext or real_ext[1:] == expected_real_ext else filename,
-        ext)
+prepend_extension = functools.partial(_change_extension, True)
+replace_extension = functools.partial(_change_extension, False)
 
 
 def check_executable(exe, args=[]):
@@ -6579,3 +6549,136 @@ def join_nonempty(*values, **kwargs):
     if from_dict is not None:
         values = (traverse_obj(from_dict, variadic(v)) for v in values)
     return delim.join(map(compat_str, filter(None, values)))
+
+
+class Namespace(object):
+    """Immutable namespace"""
+
+    def __init__(self, **kw_attr):
+        self.__dict__.update(kw_attr)
+
+    def __iter__(self):
+        return iter(self.__dict__.values())
+
+    @property
+    def items_(self):
+        return self.__dict__.items()
+
+
+MEDIA_EXTENSIONS = Namespace(
+    common_video=('avi', 'flv', 'mkv', 'mov', 'mp4', 'webm'),
+    video=('3g2', '3gp', 'f4v', 'mk3d', 'divx', 'mpg', 'ogv', 'm4v', 'wmv'),
+    common_audio=('aiff', 'alac', 'flac', 'm4a', 'mka', 'mp3', 'ogg', 'opus', 'wav'),
+    audio=('aac', 'ape', 'asf', 'f4a', 'f4b', 'm4b', 'm4p', 'm4r', 'oga', 'ogx', 'spx', 'vorbis', 'wma', 'weba'),
+    thumbnails=('jpg', 'png', 'webp'),
+    # storyboards=('mhtml', ),
+    subtitles=('srt', 'vtt', 'ass', 'lrc', 'ttml'),
+    manifests=('f4f', 'f4m', 'm3u8', 'smil', 'mpd'),
+)
+MEDIA_EXTENSIONS.video = MEDIA_EXTENSIONS.common_video + MEDIA_EXTENSIONS.video
+MEDIA_EXTENSIONS.audio = MEDIA_EXTENSIONS.common_audio + MEDIA_EXTENSIONS.audio
+
+KNOWN_EXTENSIONS = (
+    MEDIA_EXTENSIONS.video + MEDIA_EXTENSIONS.audio
+    + MEDIA_EXTENSIONS.manifests
+)
+
+
+class _UnsafeExtensionError(Exception):
+    """
+    Mitigation exception for unwanted file overwrite/path traversal
+    This should be caught in YoutubeDL.py with a warning
+
+    Ref: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-79w7-vh3h-8g4j
+    """
+    _ALLOWED_EXTENSIONS = frozenset(itertools.chain(
+        (   # internal
+            'description',
+            'json',
+            'meta',
+            'orig',
+            'part',
+            'temp',
+            'uncut',
+            'unknown_video',
+            'ytdl',
+        ),
+        # video
+        MEDIA_EXTENSIONS.video, (
+            'avif',
+            'ismv',
+            'm2ts',
+            'm4s',
+            'mng',
+            'mpeg',
+            'qt',
+            'swf',
+            'ts',
+            'vp9',
+            'wvm',
+        ),
+        # audio
+        MEDIA_EXTENSIONS.audio, (
+            'isma',
+            'mid',
+            'mpga',
+            'ra',
+        ),
+        # image
+        MEDIA_EXTENSIONS.thumbnails, (
+            'bmp',
+            'gif',
+            'ico',
+            'heic',
+            'jng',
+            'jpeg',
+            'jxl',
+            'svg',
+            'tif',
+            'wbmp',
+        ),
+        # subtitle
+        MEDIA_EXTENSIONS.subtitles, (
+            'dfxp',
+            'fs',
+            'ismt',
+            'sami',
+            'scc',
+            'ssa',
+            'tt',
+        ),
+        # others
+        MEDIA_EXTENSIONS.manifests,
+        (
+            # not used in yt-dl
+            # *MEDIA_EXTENSIONS.storyboards,
+            # 'desktop',
+            # 'ism',
+            # 'm3u',
+            # 'sbv',
+            # 'swp',
+            # 'url',
+            # 'webloc',
+            # 'xml',
+        )))
+
+    def __init__(self, extension):
+        super(_UnsafeExtensionError, self).__init__('unsafe file extension: {0!r}'.format(extension))
+        self.extension = extension
+
+    @classmethod
+    def sanitize_extension(cls, extension, **kwargs):
+        # ... /, *, prepend=False
+        prepend = kwargs.get('prepend', False)
+
+        if '/' in extension or '\\' in extension:
+            raise cls(extension)
+
+        if not prepend:
+            last = extension.rpartition('.')[-1]
+            if last == 'bin':
+                extension = last = 'unknown_video'
+            if last.lower() not in cls._ALLOWED_EXTENSIONS:
+                raise cls(extension)
+
+        return extension

From 37cea84f775129ad715b9bcd617251c831fcc980 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 2 Jul 2024 14:54:25 +0100
Subject: [PATCH 774/860] [core,utils] Support unpublicised
 `--no-check-extensions`

---
 youtube_dl/__init__.py | 4 ++++
 youtube_dl/options.py  | 4 ++++
 youtube_dl/utils.py    | 6 ++++--
 3 files changed, 12 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index cc8285eba..06bdfb689 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -21,6 +21,7 @@ from .compat import (
     workaround_optparse_bug9161,
 )
 from .utils import (
+    _UnsafeExtensionError,
     DateRange,
     decodeOption,
     DEFAULT_OUTTMPL,
@@ -173,6 +174,9 @@ def _real_main(argv=None):
     if opts.ap_mso and opts.ap_mso not in MSO_INFO:
         parser.error('Unsupported TV Provider, use --ap-list-mso to get a list of supported TV Providers')
 
+    if opts.no_check_extensions:
+        _UnsafeExtensionError.lenient = True
+
     def parse_retries(retries):
         if retries in ('inf', 'infinite'):
             parsed_retries = float('inf')
diff --git a/youtube_dl/options.py b/youtube_dl/options.py
index 434f520d3..61705d1f0 100644
--- a/youtube_dl/options.py
+++ b/youtube_dl/options.py
@@ -533,6 +533,10 @@ def parseOpts(overrideArguments=None):
         '--no-check-certificate',
         action='store_true', dest='no_check_certificate', default=False,
         help='Suppress HTTPS certificate validation')
+    workarounds.add_option(
+        '--no-check-extensions',
+        action='store_true', dest='no_check_extensions', default=False,
+        help='Suppress file extension validation')
     workarounds.add_option(
         '--prefer-insecure',
         '--prefer-unsecure', action='store_true', dest='prefer_insecure',
diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index df203b97a..3ec9d3811 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -6587,7 +6587,6 @@ KNOWN_EXTENSIONS = (
 class _UnsafeExtensionError(Exception):
     """
     Mitigation exception for unwanted file overwrite/path traversal
-    This should be caught in YoutubeDL.py with a warning
 
     Ref: https://github.com/yt-dlp/yt-dlp/security/advisories/GHSA-79w7-vh3h-8g4j
     """
@@ -6666,6 +6665,9 @@ class _UnsafeExtensionError(Exception):
         super(_UnsafeExtensionError, self).__init__('unsafe file extension: {0!r}'.format(extension))
         self.extension = extension
 
+    # support --no-check-extensions
+    lenient = False
+
     @classmethod
     def sanitize_extension(cls, extension, **kwargs):
         # ... /, *, prepend=False
@@ -6678,7 +6680,7 @@ class _UnsafeExtensionError(Exception):
             last = extension.rpartition('.')[-1]
             if last == 'bin':
                 extension = last = 'unknown_video'
-            if last.lower() not in cls._ALLOWED_EXTENSIONS:
+            if not (cls.lenient or last.lower() in cls._ALLOWED_EXTENSIONS):
                 raise cls(extension)
 
         return extension

From f4b47754d9716dbe61372d75e85e965328335ec9 Mon Sep 17 00:00:00 2001
From: Sergey Musatov <akapulse@yandex.ru>
Date: Sat, 6 Jul 2024 13:04:36 +0300
Subject: [PATCH 775/860] [YandexMusic] Download music in High Quality (320
 Kbit/s)

PR #31159
---
 youtube_dl/extractor/yandexmusic.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 84969f8e1..55d4fb5a0 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -120,6 +120,7 @@ class YandexMusicTrackIE(YandexMusicBaseIE):
         download_data = self._download_json(
             'https://music.yandex.ru/api/v2.1/handlers/track/%s:%s/web-album_track-track-track-main/download/m' % (track_id, album_id),
             track_id, 'Downloading track location url JSON',
+            query={'hq': 1},
             headers={'X-Retpath-Y': url})
 
         fd_data = self._download_json(

From 36801c62df733cfa4f74b607532e012900d8c77f Mon Sep 17 00:00:00 2001
From: unkernet <unkernet@gmail.com>
Date: Sun, 7 Jul 2024 21:18:33 +0200
Subject: [PATCH 776/860] [YandexMusic] Save track version in the title field

PR #32837
* Add track version to track title
---
 youtube_dl/extractor/yandexmusic.py | 22 ++++++++++++++++++++++
 1 file changed, 22 insertions(+)

diff --git a/youtube_dl/extractor/yandexmusic.py b/youtube_dl/extractor/yandexmusic.py
index 55d4fb5a0..8da5b430f 100644
--- a/youtube_dl/extractor/yandexmusic.py
+++ b/youtube_dl/extractor/yandexmusic.py
@@ -106,6 +106,25 @@ class YandexMusicTrackIE(YandexMusicBaseIE):
     }, {
         'url': 'http://music.yandex.com/album/540508/track/4878838',
         'only_matching': True,
+    }, {
+        'url': 'https://music.yandex.ru/album/16302456/track/85430762',
+        'md5': '11b8d50ab03b57738deeaadf661a0a48',
+        'info_dict': {
+            'id': '85430762',
+            'ext': 'mp3',
+            'abr': 128,
+            'title': 'Haddadi Von Engst, Phonic Youth, Super Flu - Til The End (Super Flu Remix)',
+            'filesize': int,
+            'duration': 431.14,
+            'track': 'Til The End (Super Flu Remix)',
+            'album': 'Til The End',
+            'album_artist': 'Haddadi Von Engst, Phonic Youth',
+            'artist': 'Haddadi Von Engst, Phonic Youth, Super Flu',
+            'release_year': 2021,
+            'genre': 'house',
+            'disc_number': 1,
+            'track_number': 2,
+        }
     }]
 
     def _real_extract(self, url):
@@ -116,6 +135,9 @@ class YandexMusicTrackIE(YandexMusicBaseIE):
             'track', tld, url, track_id, 'Downloading track JSON',
             {'track': '%s:%s' % (track_id, album_id)})['track']
         track_title = track['title']
+        track_version = track.get('version')
+        if track_version:
+            track_title = '%s (%s)' % (track_title, track_version)
 
         download_data = self._download_json(
             'https://music.yandex.ru/api/v2.1/handlers/track/%s:%s/web-album_track-track-track-main/download/m' % (track_id, album_id),

From a452f9437c8a3048f75fc12f75bcfd3eed78430f Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 7 Jul 2024 20:43:10 +0100
Subject: [PATCH 777/860] [core] Fix PR #32830 for fixed extensionless output
 template

---
 youtube_dl/YoutubeDL.py | 29 +++++++++++++++++++----------
 1 file changed, 19 insertions(+), 10 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index c19501915..9e5620eef 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -139,8 +139,8 @@ def _catch_unsafe_file_extension(func):
         except _UnsafeExtensionError as error:
             self.report_error(
                 '{0} found; to avoid damaging your system, this value is disallowed.'
-                ' If you believe this is an error{1}').format(
-                    error.message, bug_reports_message(','))
+                ' If you believe this is an error{1}'.format(
+                    error_to_compat_str(error), bug_reports_message(',')))
 
     return wrapper
 
@@ -2114,18 +2114,26 @@ class YoutubeDL(object):
                         # TODO: Check acodec/vcodec
                         return False
 
-                    filename_real_ext = os.path.splitext(filename)[1][1:]
-                    filename_wo_ext = (
-                        os.path.splitext(filename)[0]
-                        if filename_real_ext == info_dict['ext']
-                        else filename)
+                    exts = [info_dict['ext']]
                     requested_formats = info_dict['requested_formats']
                     if self.params.get('merge_output_format') is None and not compatible_formats(requested_formats):
                         info_dict['ext'] = 'mkv'
                         self.report_warning(
                             'Requested formats are incompatible for merge and will be merged into mkv.')
+                    exts.append(info_dict['ext'])
+
                     # Ensure filename always has a correct extension for successful merge
-                    filename = '%s.%s' % (filename_wo_ext, info_dict['ext'])
+                    def correct_ext(filename, ext=exts[1]):
+                        if filename == '-':
+                            return filename
+                        f_name, f_real_ext = os.path.splitext(filename)
+                        f_real_ext = f_real_ext[1:]
+                        filename_wo_ext = f_name if f_real_ext in exts else filename
+                        if ext is None:
+                            ext = f_real_ext or None
+                        return join_nonempty(filename_wo_ext, ext, delim='.')
+
+                    filename = correct_ext(filename)
                     if os.path.exists(encodeFilename(filename)):
                         self.to_screen(
                             '[download] %s has already been downloaded and '
@@ -2135,8 +2143,9 @@ class YoutubeDL(object):
                             new_info = dict(info_dict)
                             new_info.update(f)
                             fname = prepend_extension(
-                                self.prepare_filename(new_info),
-                                'f%s' % f['format_id'], new_info['ext'])
+                                correct_ext(
+                                    self.prepare_filename(new_info), new_info['ext']),
+                                'f%s' % (f['format_id'],), new_info['ext'])
                             if not ensure_dir_exists(fname):
                                 return
                             downloaded.append(fname)

From 43a74c5fa5e43ee563b1488634585cb0bf02ea24 Mon Sep 17 00:00:00 2001
From: bashonly <bashonly@protonmail.com>
Date: Fri, 5 Jul 2024 13:41:30 -0500
Subject: [PATCH 778/860] [core] Address gaps in allowed extensions

Adds some extensions missing in 46521096433aceaa41b4caa845bed22ca6f377ce
(from yt-dlp/yt-dlp#10362)

Authored by: bashonly
Co-authored by: dirkf
---
 youtube_dl/utils.py | 37 ++++++++++++++++++++++++++++++++++---
 1 file changed, 34 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index 3ec9d3811..ac1e78002 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -6604,27 +6604,53 @@ class _UnsafeExtensionError(Exception):
         ),
         # video
         MEDIA_EXTENSIONS.video, (
-            'avif',
+            'asx',
             'ismv',
+            'm2t',
             'm2ts',
+            'm2v',
             'm4s',
             'mng',
+            'mp2v',
+            'mp4v',
+            'mpe',
             'mpeg',
+            'mpeg1',
+            'mpeg2',
+            'mpeg4',
+            'mxf',
+            'ogm',
             'qt',
+            'rm',
             'swf',
             'ts',
+            'vob',
             'vp9',
-            'wvm',
         ),
         # audio
         MEDIA_EXTENSIONS.audio, (
+            '3ga',
+            'ac3',
+            'adts',
+            'aif',
+            'au',
+            'dts',
             'isma',
+            'it',
             'mid',
+            'mod',
             'mpga',
+            'mp1',
+            'mp2',
+            'mp4a',
+            'mpa',
             'ra',
+            'shn',
+            'xm',
         ),
         # image
         MEDIA_EXTENSIONS.thumbnails, (
+            'avif',
             'bmp',
             'gif',
             'ico',
@@ -6634,6 +6660,7 @@ class _UnsafeExtensionError(Exception):
             'jxl',
             'svg',
             'tif',
+            'tiff',
             'wbmp',
         ),
         # subtitle
@@ -6641,10 +6668,15 @@ class _UnsafeExtensionError(Exception):
             'dfxp',
             'fs',
             'ismt',
+            'json3',
             'sami',
             'scc',
+            'srv1',
+            'srv2',
+            'srv3',
             'ssa',
             'tt',
+            'xml',
         ),
         # others
         MEDIA_EXTENSIONS.manifests,
@@ -6658,7 +6690,6 @@ class _UnsafeExtensionError(Exception):
             # 'swp',
             # 'url',
             # 'webloc',
-            # 'xml',
         )))
 
     def __init__(self, extension):

From 756f6b45c7c9a77f0fa912f882afe7e04878a01b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 10 Jul 2024 17:58:38 +0100
Subject: [PATCH 779/860] [jsinterp] Re-align JSInterp and tests (esp.) with
 yt-dlp Thx: various yt-dlp authors

---
 test/test_jsinterp.py  | 675 +++++++++++++++--------------------------
 youtube_dl/jsinterp.py |  53 ++--
 2 files changed, 268 insertions(+), 460 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index da8e98020..86d7f0d46 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -11,194 +11,146 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 import math
 import re
 
+from youtube_dl.compat import compat_str
 from youtube_dl.jsinterp import JS_Undefined, JSInterpreter
 
+NaN = object()
+
 
 class TestJSInterpreter(unittest.TestCase):
+    def _test(self, jsi_or_code, expected, func='f', args=()):
+        if isinstance(jsi_or_code, compat_str):
+            jsi_or_code = JSInterpreter(jsi_or_code)
+        got = jsi_or_code.call_function(func, *args)
+        if expected is NaN:
+            self.assertTrue(math.isnan(got), '{0} is not NaN'.format(got))
+        else:
+            self.assertEqual(got, expected)
+
     def test_basic(self):
-        jsi = JSInterpreter('function x(){;}')
-        self.assertEqual(jsi.call_function('x'), None)
-        self.assertEqual(repr(jsi.extract_function('x')), 'F<x>')
+        jsi = JSInterpreter('function f(){;}')
+        self.assertEqual(repr(jsi.extract_function('f')), 'F<f>')
+        self._test(jsi, None)
 
-        jsi = JSInterpreter('function x3(){return 42;}')
-        self.assertEqual(jsi.call_function('x3'), 42)
-
-        jsi = JSInterpreter('function x3(){42}')
-        self.assertEqual(jsi.call_function('x3'), None)
-
-        jsi = JSInterpreter('var x5 = function(){return 42;}')
-        self.assertEqual(jsi.call_function('x5'), 42)
-
-    def test_calc(self):
-        jsi = JSInterpreter('function x4(a){return 2*a+1;}')
-        self.assertEqual(jsi.call_function('x4', 3), 7)
+        self._test('function f(){return 42;}', 42)
+        self._test('function f(){42}', None)
+        self._test('var f = function(){return 42;}', 42)
 
     def test_add(self):
-        jsi = JSInterpreter('function f(){return 42 + 7;}')
-        self.assertEqual(jsi.call_function('f'), 49)
-        jsi = JSInterpreter('function f(){return 42 + undefined;}')
-        self.assertTrue(math.isnan(jsi.call_function('f')))
-        jsi = JSInterpreter('function f(){return 42 + null;}')
-        self.assertEqual(jsi.call_function('f'), 42)
+        self._test('function f(){return 42 + 7;}', 49)
+        self._test('function f(){return 42 + undefined;}', NaN)
+        self._test('function f(){return 42 + null;}', 42)
 
     def test_sub(self):
-        jsi = JSInterpreter('function f(){return 42 - 7;}')
-        self.assertEqual(jsi.call_function('f'), 35)
-        jsi = JSInterpreter('function f(){return 42 - undefined;}')
-        self.assertTrue(math.isnan(jsi.call_function('f')))
-        jsi = JSInterpreter('function f(){return 42 - null;}')
-        self.assertEqual(jsi.call_function('f'), 42)
+        self._test('function f(){return 42 - 7;}', 35)
+        self._test('function f(){return 42 - undefined;}', NaN)
+        self._test('function f(){return 42 - null;}', 42)
 
     def test_mul(self):
-        jsi = JSInterpreter('function f(){return 42 * 7;}')
-        self.assertEqual(jsi.call_function('f'), 294)
-        jsi = JSInterpreter('function f(){return 42 * undefined;}')
-        self.assertTrue(math.isnan(jsi.call_function('f')))
-        jsi = JSInterpreter('function f(){return 42 * null;}')
-        self.assertEqual(jsi.call_function('f'), 0)
+        self._test('function f(){return 42 * 7;}', 294)
+        self._test('function f(){return 42 * undefined;}', NaN)
+        self._test('function f(){return 42 * null;}', 0)
 
     def test_div(self):
         jsi = JSInterpreter('function f(a, b){return a / b;}')
-        self.assertTrue(math.isnan(jsi.call_function('f', 0, 0)))
-        self.assertTrue(math.isnan(jsi.call_function('f', JS_Undefined, 1)))
-        self.assertTrue(math.isinf(jsi.call_function('f', 2, 0)))
-        self.assertEqual(jsi.call_function('f', 0, 3), 0)
+        self._test(jsi, NaN, args=(0, 0))
+        self._test(jsi, NaN, args=(JS_Undefined, 1))
+        self._test(jsi, float('inf'), args=(2, 0))
+        self._test(jsi, 0, args=(0, 3))
 
     def test_mod(self):
-        jsi = JSInterpreter('function f(){return 42 % 7;}')
-        self.assertEqual(jsi.call_function('f'), 0)
-        jsi = JSInterpreter('function f(){return 42 % 0;}')
-        self.assertTrue(math.isnan(jsi.call_function('f')))
-        jsi = JSInterpreter('function f(){return 42 % undefined;}')
-        self.assertTrue(math.isnan(jsi.call_function('f')))
+        self._test('function f(){return 42 % 7;}', 0)
+        self._test('function f(){return 42 % 0;}', NaN)
+        self._test('function f(){return 42 % undefined;}', NaN)
 
     def test_exp(self):
-        jsi = JSInterpreter('function f(){return 42 ** 2;}')
-        self.assertEqual(jsi.call_function('f'), 1764)
-        jsi = JSInterpreter('function f(){return 42 ** undefined;}')
-        self.assertTrue(math.isnan(jsi.call_function('f')))
-        jsi = JSInterpreter('function f(){return 42 ** null;}')
-        self.assertEqual(jsi.call_function('f'), 1)
-        jsi = JSInterpreter('function f(){return undefined ** 42;}')
-        self.assertTrue(math.isnan(jsi.call_function('f')))
+        self._test('function f(){return 42 ** 2;}', 1764)
+        self._test('function f(){return 42 ** undefined;}', NaN)
+        self._test('function f(){return 42 ** null;}', 1)
+        self._test('function f(){return undefined ** 42;}', NaN)
+
+    def test_calc(self):
+        self._test('function f(a){return 2*a+1;}', 7, args=[3])
 
     def test_empty_return(self):
-        jsi = JSInterpreter('function f(){return; y()}')
-        self.assertEqual(jsi.call_function('f'), None)
+        self._test('function f(){return; y()}', None)
 
     def test_morespace(self):
-        jsi = JSInterpreter('function x (a) { return 2 * a + 1 ; }')
-        self.assertEqual(jsi.call_function('x', 3), 7)
-
-        jsi = JSInterpreter('function f () { x =  2  ; return x; }')
-        self.assertEqual(jsi.call_function('f'), 2)
+        self._test('function f (a) { return 2 * a + 1 ; }', 7, args=[3])
+        self._test('function f () { x =  2  ; return x; }', 2)
 
     def test_strange_chars(self):
-        jsi = JSInterpreter('function $_xY1 ($_axY1) { var $_axY2 = $_axY1 + 1; return $_axY2; }')
-        self.assertEqual(jsi.call_function('$_xY1', 20), 21)
+        self._test('function $_xY1 ($_axY1) { var $_axY2 = $_axY1 + 1; return $_axY2; }',
+                   21, args=[20], func='$_xY1')
 
     def test_operators(self):
-        jsi = JSInterpreter('function f(){return 1 << 5;}')
-        self.assertEqual(jsi.call_function('f'), 32)
-
-        jsi = JSInterpreter('function f(){return 2 ** 5}')
-        self.assertEqual(jsi.call_function('f'), 32)
-
-        jsi = JSInterpreter('function f(){return 19 & 21;}')
-        self.assertEqual(jsi.call_function('f'), 17)
-
-        jsi = JSInterpreter('function f(){return 11 >> 2;}')
-        self.assertEqual(jsi.call_function('f'), 2)
-
-        jsi = JSInterpreter('function f(){return []? 2+3: 4;}')
-        self.assertEqual(jsi.call_function('f'), 5)
-
-        jsi = JSInterpreter('function f(){return 1 == 2}')
-        self.assertEqual(jsi.call_function('f'), False)
-
-        jsi = JSInterpreter('function f(){return 0 && 1 || 2;}')
-        self.assertEqual(jsi.call_function('f'), 2)
-
-        jsi = JSInterpreter('function f(){return 0 ?? 42;}')
-        self.assertEqual(jsi.call_function('f'), 0)
-
-        jsi = JSInterpreter('function f(){return "life, the universe and everything" < 42;}')
-        self.assertFalse(jsi.call_function('f'))
+        self._test('function f(){return 1 << 5;}', 32)
+        self._test('function f(){return 2 ** 5}', 32)
+        self._test('function f(){return 19 & 21;}', 17)
+        self._test('function f(){return 11 >> 2;}', 2)
+        self._test('function f(){return []? 2+3: 4;}', 5)
+        self._test('function f(){return 1 == 2}', False)
+        self._test('function f(){return 0 && 1 || 2;}', 2)
+        self._test('function f(){return 0 ?? 42;}', 0)
+        self._test('function f(){return "life, the universe and everything" < 42;}', False)
+        # https://github.com/ytdl-org/youtube-dl/issues/32815
+        self._test('function f(){return 0  - 7 * - 6;}', 42)
 
     def test_array_access(self):
-        jsi = JSInterpreter('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}')
-        self.assertEqual(jsi.call_function('f'), [5, 2, 7])
+        self._test('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}', [5, 2, 7])
 
     def test_parens(self):
-        jsi = JSInterpreter('function f(){return (1) + (2) * ((( (( (((((3)))))) )) ));}')
-        self.assertEqual(jsi.call_function('f'), 7)
-
-        jsi = JSInterpreter('function f(){return (1 + 2) * 3;}')
-        self.assertEqual(jsi.call_function('f'), 9)
+        self._test('function f(){return (1) + (2) * ((( (( (((((3)))))) )) ));}', 7)
+        self._test('function f(){return (1 + 2) * 3;}', 9)
 
     def test_quotes(self):
-        jsi = JSInterpreter(r'function f(){return "a\"\\("}')
-        self.assertEqual(jsi.call_function('f'), r'a"\(')
+        self._test(r'function f(){return "a\"\\("}', r'a"\(')
 
     def test_assignments(self):
-        jsi = JSInterpreter('function f(){var x = 20; x = 30 + 1; return x;}')
-        self.assertEqual(jsi.call_function('f'), 31)
-
-        jsi = JSInterpreter('function f(){var x = 20; x += 30 + 1; return x;}')
-        self.assertEqual(jsi.call_function('f'), 51)
-
-        jsi = JSInterpreter('function f(){var x = 20; x -= 30 + 1; return x;}')
-        self.assertEqual(jsi.call_function('f'), -11)
+        self._test('function f(){var x = 20; x = 30 + 1; return x;}', 31)
+        self._test('function f(){var x = 20; x += 30 + 1; return x;}', 51)
+        self._test('function f(){var x = 20; x -= 30 + 1; return x;}', -11)
 
+    @unittest.skip('Not yet fully implemented')
     def test_comments(self):
-        'Skipping: Not yet fully implemented'
-        return
-        jsi = JSInterpreter('''
-        function x() {
-            var x = /* 1 + */ 2;
-            var y = /* 30
-            * 40 */ 50;
-            return x + y;
-        }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 52)
+        self._test('''
+            function f() {
+                var x = /* 1 + */ 2;
+                var y = /* 30
+                * 40 */ 50;
+                return x + y;
+            }
+        ''', 52)
 
-        jsi = JSInterpreter('''
-        function f() {
-            var x = "/*";
-            var y = 1 /* comment */ + 2;
-            return y;
-        }
-        ''')
-        self.assertEqual(jsi.call_function('f'), 3)
+        self._test('''
+            function f() {
+                var x = "/*";
+                var y = 1 /* comment */ + 2;
+                return y;
+            }
+        ''', 3)
 
     def test_precedence(self):
-        jsi = JSInterpreter('''
-        function x() {
-            var a = [10, 20, 30, 40, 50];
-            var b = 6;
-            a[0]=a[b%a.length];
-            return a;
-        }''')
-        self.assertEqual(jsi.call_function('x'), [20, 20, 30, 40, 50])
+        self._test('''
+            function f() {
+                var a = [10, 20, 30, 40, 50];
+                var b = 6;
+                a[0]=a[b%a.length];
+                return a;
+            }
+        ''', [20, 20, 30, 40, 50])
 
     def test_builtins(self):
-        jsi = JSInterpreter('''
-        function x() { return NaN }
-        ''')
-        self.assertTrue(math.isnan(jsi.call_function('x')))
+        self._test('function f() { return NaN }', NaN)
 
     def test_Date(self):
-        jsi = JSInterpreter('''
-        function x(dt) { return new Date(dt) - 0; }
-        ''')
-        self.assertEqual(jsi.call_function('x', 'Wednesday 31 December 1969 18:01:26 MDT'), 86000)
+        self._test('function f() { return new Date("Wednesday 31 December 1969 18:01:26 MDT") - 0; }', 86000)
 
+        jsi = JSInterpreter('function f(dt) { return new Date(dt) - 0; }')
         # date format m/d/y
-        self.assertEqual(jsi.call_function('x', '12/31/1969 18:01:26 MDT'), 86000)
-
+        self._test(jsi, 86000, args=['12/31/1969 18:01:26 MDT'])
         # epoch 0
-        self.assertEqual(jsi.call_function('x', '1 January 1970 00:00:00 UTC'), 0)
+        self._test(jsi, 0, args=['1 January 1970 00:00:00 UTC'])
 
     def test_call(self):
         jsi = JSInterpreter('''
@@ -206,179 +158,115 @@ class TestJSInterpreter(unittest.TestCase):
         function y(a) { return x() + (a?a:0); }
         function z() { return y(3); }
         ''')
-        self.assertEqual(jsi.call_function('z'), 5)
-        self.assertEqual(jsi.call_function('y'), 2)
+        self._test(jsi, 5, func='z')
+        self._test(jsi, 2, func='y')
 
     def test_if(self):
-        jsi = JSInterpreter('''
-        function x() {
+        self._test('''
+            function f() {
             let a = 9;
             if (0==0) {a++}
             return a
-        }''')
-        self.assertEqual(jsi.call_function('x'), 10)
+            }
+        ''', 10)
 
-        jsi = JSInterpreter('''
-        function x() {
+        self._test('''
+            function f() {
             if (0==0) {return 10}
-        }''')
-        self.assertEqual(jsi.call_function('x'), 10)
+            }
+        ''', 10)
 
-        jsi = JSInterpreter('''
-        function x() {
+        self._test('''
+            function f() {
             if (0!=0) {return 1}
             else {return 10}
-        }''')
-        self.assertEqual(jsi.call_function('x'), 10)
-
-        """  # Unsupported
-        jsi = JSInterpreter('''
-        function x() {
-            if (0!=0) return 1;
-            else {return 10}
-        }''')
-        self.assertEqual(jsi.call_function('x'), 10)
-        """
+            }
+        ''', 10)
 
     def test_elseif(self):
-        jsi = JSInterpreter('''
-        function x() {
-            if (0!=0) {return 1}
-            else if (1==0) {return 2}
-            else {return 10}
-        }''')
-        self.assertEqual(jsi.call_function('x'), 10)
-
-        """  # Unsupported
-        jsi = JSInterpreter('''
-        function x() {
-            if (0!=0) return 1;
-            else if (1==0) {return 2}
-            else {return 10}
-        }''')
-        self.assertEqual(jsi.call_function('x'), 10)
-        # etc
-        """
+        self._test('''
+            function f() {
+                if (0!=0) {return 1}
+                else if (1==0) {return 2}
+                else {return 10}
+            }
+        ''', 10)
 
     def test_for_loop(self):
-        # function x() { a=0; for (i=0; i-10; i++) {a++} a }
-        jsi = JSInterpreter('''
-        function x() { a=0; for (i=0; i-10; i++) {a++} return a }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 10)
+        self._test('function f() { a=0; for (i=0; i-10; i++) {a++} return a }', 10)
 
     def test_while_loop(self):
-        # function x() { a=0; while (a<10) {a++} a }
-        jsi = JSInterpreter('''
-        function x() { a=0; while (a<10) {a++} return a }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 10)
+        self._test('function f() { a=0; while (a<10) {a++} return a }', 10)
 
     def test_switch(self):
         jsi = JSInterpreter('''
-        function x(f) { switch(f){
-            case 1:f+=1;
-            case 2:f+=2;
-            case 3:f+=3;break;
-            case 4:f+=4;
-            default:f=0;
-        } return f }
+            function f(x) { switch(x){
+                case 1:x+=1;
+                case 2:x+=2;
+                case 3:x+=3;break;
+                case 4:x+=4;
+                default:x=0;
+            } return x }
         ''')
-        self.assertEqual(jsi.call_function('x', 1), 7)
-        self.assertEqual(jsi.call_function('x', 3), 6)
-        self.assertEqual(jsi.call_function('x', 5), 0)
+        self._test(jsi, 7, args=[1])
+        self._test(jsi, 6, args=[3])
+        self._test(jsi, 0, args=[5])
 
     def test_switch_default(self):
         jsi = JSInterpreter('''
-        function x(f) { switch(f){
-            case 2: f+=2;
-            default: f-=1;
-            case 5:
-            case 6: f+=6;
-            case 0: break;
-            case 1: f+=1;
-        } return f }
+            function f(x) { switch(x){
+                case 2: x+=2;
+                default: x-=1;
+                case 5:
+                case 6: x+=6;
+                case 0: break;
+                case 1: x+=1;
+            } return x }
         ''')
-        self.assertEqual(jsi.call_function('x', 1), 2)
-        self.assertEqual(jsi.call_function('x', 5), 11)
-        self.assertEqual(jsi.call_function('x', 9), 14)
+        self._test(jsi, 2, args=[1])
+        self._test(jsi, 11, args=[5])
+        self._test(jsi, 14, args=[9])
 
     def test_try(self):
-        jsi = JSInterpreter('''
-        function x() { try{return 10} catch(e){return 5} }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 10)
+        self._test('function f() { try{return 10} catch(e){return 5} }', 10)
 
     def test_catch(self):
-        jsi = JSInterpreter('''
-        function x() { try{throw 10} catch(e){return 5} }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 5)
+        self._test('function f() { try{throw 10} catch(e){return 5} }', 5)
 
     def test_finally(self):
-        jsi = JSInterpreter('''
-        function x() { try{throw 10} finally {return 42} }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 42)
-        jsi = JSInterpreter('''
-        function x() { try{throw 10} catch(e){return 5} finally {return 42} }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 42)
+        self._test('function f() { try{throw 10} finally {return 42} }', 42)
+        self._test('function f() { try{throw 10} catch(e){return 5} finally {return 42} }', 42)
 
     def test_nested_try(self):
-        jsi = JSInterpreter('''
-        function x() {try {
-            try{throw 10} finally {throw 42}
+        self._test('''
+            function f() {try {
+                try{throw 10} finally {throw 42}
             } catch(e){return 5} }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 5)
+        ''', 5)
 
     def test_for_loop_continue(self):
-        jsi = JSInterpreter('''
-        function x() { a=0; for (i=0; i-10; i++) { continue; a++ } return a }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 0)
+        self._test('function f() { a=0; for (i=0; i-10; i++) { continue; a++ } return a }', 0)
 
     def test_for_loop_break(self):
-        jsi = JSInterpreter('''
-        function x() { a=0; for (i=0; i-10; i++) { break; a++ } return a }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 0)
+        self._test('function f() { a=0; for (i=0; i-10; i++) { break; a++ } return a }', 0)
 
     def test_for_loop_try(self):
-        jsi = JSInterpreter('''
-        function x() {
-            for (i=0; i-10; i++) { try { if (i == 5) throw i} catch {return 10} finally {break} };
-            return 42 }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 42)
+        self._test('''
+            function f() {
+                for (i=0; i-10; i++) { try { if (i == 5) throw i} catch {return 10} finally {break} };
+                return 42 }
+        ''', 42)
 
     def test_literal_list(self):
-        jsi = JSInterpreter('''
-        function x() { return [1, 2, "asdf", [5, 6, 7]][3] }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [5, 6, 7])
+        self._test('function f() { return [1, 2, "asdf", [5, 6, 7]][3] }', [5, 6, 7])
 
     def test_comma(self):
-        jsi = JSInterpreter('''
-        function x() { a=5; a -= 1, a+=3; return a }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 7)
-        jsi = JSInterpreter('''
-        function x() { a=5; return (a -= 1, a+=3, a); }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 7)
-
-        jsi = JSInterpreter('''
-        function x() { return (l=[0,1,2,3], function(a, b){return a+b})((l[1], l[2]), l[3]) }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 5)
+        self._test('function f() { a=5; a -= 1, a+=3; return a }', 7)
+        self._test('function f() { a=5; return (a -= 1, a+=3, a); }', 7)
+        self._test('function f() { return (l=[0,1,2,3], function(a, b){return a+b})((l[1], l[2]), l[3]) }', 5)
 
     def test_void(self):
-        jsi = JSInterpreter('''
-        function x() { return void 42; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), None)
+        self._test('function f() { return void 42; }', None)
 
     def test_return_function(self):
         jsi = JSInterpreter('''
@@ -387,110 +275,60 @@ class TestJSInterpreter(unittest.TestCase):
         self.assertEqual(jsi.call_function('x')([]), 1)
 
     def test_null(self):
-        jsi = JSInterpreter('''
-        function x() { return null; }
-        ''')
-        self.assertIs(jsi.call_function('x'), None)
-
-        jsi = JSInterpreter('''
-        function x() { return [null > 0, null < 0, null == 0, null === 0]; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [False, False, False, False])
-
-        jsi = JSInterpreter('''
-        function x() { return [null >= 0, null <= 0]; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [True, True])
+        self._test('function f() { return null; }', None)
+        self._test('function f() { return [null > 0, null < 0, null == 0, null === 0]; }',
+                   [False, False, False, False])
+        self._test('function f() { return [null >= 0, null <= 0]; }', [True, True])
 
     def test_undefined(self):
-        jsi = JSInterpreter('''
-        function x() { return undefined === undefined; }
-        ''')
-        self.assertTrue(jsi.call_function('x'))
+        self._test('function f() { return undefined === undefined; }', True)
+        self._test('function f() { return undefined; }', JS_Undefined)
+        self._test('function f() {return undefined ?? 42; }', 42)
+        self._test('function f() { let v; return v; }', JS_Undefined)
+        self._test('function f() { let v; return v**0; }', 1)
+        self._test('function f() { let v; return [v>42, v<=42, v&&42, 42&&v]; }',
+                   [False, False, JS_Undefined, JS_Undefined])
+
+        self._test('''
+            function f() { return [
+                undefined === undefined,
+                undefined == undefined,
+                undefined == null
+            ]; }
+        ''', [True] * 3)
+        self._test('''
+            function f() { return [
+                undefined < undefined,
+                undefined > undefined,
+                undefined === 0,
+                undefined == 0,
+                undefined < 0,
+                undefined > 0,
+                undefined >= 0,
+                undefined <= 0,
+                undefined > null,
+                undefined < null,
+                undefined === null
+            ]; }
+        ''', [False] * 11)
 
         jsi = JSInterpreter('''
-        function x() { return undefined; }
-        ''')
-        self.assertIs(jsi.call_function('x'), JS_Undefined)
-
-        jsi = JSInterpreter('''
-        function x() { let v; return v; }
-        ''')
-        self.assertIs(jsi.call_function('x'), JS_Undefined)
-
-        jsi = JSInterpreter('''
-        function x() { return [undefined === undefined, undefined == undefined, undefined < undefined, undefined > undefined]; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [True, True, False, False])
-
-        jsi = JSInterpreter('''
-        function x() { return [undefined === 0, undefined == 0, undefined < 0, undefined > 0]; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [False, False, False, False])
-
-        jsi = JSInterpreter('''
-        function x() { return [undefined >= 0, undefined <= 0]; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [False, False])
-
-        jsi = JSInterpreter('''
-        function x() { return [undefined > null, undefined < null, undefined == null, undefined === null]; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [False, False, True, False])
-
-        jsi = JSInterpreter('''
-        function x() { return [undefined === null, undefined == null, undefined < null, undefined > null]; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [False, True, False, False])
-
-        jsi = JSInterpreter('''
-        function x() { let v; return [42+v, v+42, v**42, 42**v, 0**v]; }
+            function x() { let v; return [42+v, v+42, v**42, 42**v, 0**v]; }
         ''')
         for y in jsi.call_function('x'):
             self.assertTrue(math.isnan(y))
 
-        jsi = JSInterpreter('''
-        function x() { let v; return v**0; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 1)
-
-        jsi = JSInterpreter('''
-        function x() { let v; return [v>42, v<=42, v&&42, 42&&v]; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [False, False, JS_Undefined, JS_Undefined])
-
-        jsi = JSInterpreter('function x(){return undefined ?? 42; }')
-        self.assertEqual(jsi.call_function('x'), 42)
-
     def test_object(self):
-        jsi = JSInterpreter('''
-        function x() { return {}; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), {})
-
-        jsi = JSInterpreter('''
-        function x() { let a = {m1: 42, m2: 0 }; return [a["m1"], a.m2]; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), [42, 0])
-
-        jsi = JSInterpreter('''
-        function x() { let a; return a?.qq; }
-        ''')
-        self.assertIs(jsi.call_function('x'), JS_Undefined)
-
-        jsi = JSInterpreter('''
-        function x() { let a = {m1: 42, m2: 0 }; return a?.qq; }
-        ''')
-        self.assertIs(jsi.call_function('x'), JS_Undefined)
+        self._test('function f() { return {}; }', {})
+        self._test('function f() { let a = {m1: 42, m2: 0 }; return [a["m1"], a.m2]; }', [42, 0])
+        self._test('function f() { let a; return a?.qq; }', JS_Undefined)
+        self._test('function f() { let a = {m1: 42, m2: 0 }; return a?.qq; }', JS_Undefined)
 
     def test_regex(self):
-        jsi = JSInterpreter('''
-        function x() { let a=/,,[/,913,/](,)}/; }
-        ''')
-        self.assertIs(jsi.call_function('x'), None)
+        self._test('function f() { let a=/,,[/,913,/](,)}/; }', None)
 
         jsi = JSInterpreter('''
-        function x() { let a=/,,[/,913,/](,)}/; "".replace(a, ""); return a; }
+            function x() { let a=/,,[/,913,/](,)}/; "".replace(a, ""); return a; }
         ''')
         attrs = set(('findall', 'finditer', 'match', 'scanner', 'search',
                      'split', 'sub', 'subn'))
@@ -500,94 +338,65 @@ class TestJSInterpreter(unittest.TestCase):
         self.assertSetEqual(set(dir(jsi.call_function('x'))) & attrs, attrs)
 
         jsi = JSInterpreter('''
-        function x() { let a=/,,[/,913,/](,)}/i; return a; }
+            function x() { let a=/,,[/,913,/](,)}/i; return a; }
         ''')
         self.assertEqual(jsi.call_function('x').flags & ~re.U, re.I)
 
-        jsi = JSInterpreter(r'''
-        function x() { let a="data-name".replace("data-", ""); return a }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 'name')
+        jsi = JSInterpreter(r'function f() { let a=/,][}",],()}(\[)/; return a; }')
+        self.assertEqual(jsi.call_function('f').pattern, r',][}",],()}(\[)')
 
-        jsi = JSInterpreter(r'''
-        function x() { let a="data-name".replace(new RegExp("^.+-"), ""); return a; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 'name')
+        jsi = JSInterpreter(r'function f() { let a=[/[)\\]/]; return a[0]; }')
+        self.assertEqual(jsi.call_function('f').pattern, r'[)\\]')
 
-        jsi = JSInterpreter(r'''
-        function x() { let a="data-name".replace(/^.+-/, ""); return a; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 'name')
-
-        jsi = JSInterpreter(r'''
-        function x() { let a="data-name".replace(/a/g, "o"); return a; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 'doto-nome')
-
-        jsi = JSInterpreter(r'''
-        function x() { let a="data-name".replaceAll("a", "o"); return a; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 'doto-nome')
-
-        jsi = JSInterpreter(r'''
-        function x() { let a=[/[)\\]/]; return a[0]; }
-        ''')
-        self.assertEqual(jsi.call_function('x').pattern, r'[)\\]')
-
-        """  # fails
-        jsi = JSInterpreter(r'''
-        function x() { let a=100; a/=/[0-9]+/.exec('divide by 20 today')[0]; }
-        ''')
-        self.assertEqual(jsi.call_function('x'), 5)
-        """
+    def test_replace(self):
+        self._test('function f() { let a="data-name".replace("data-", ""); return a }',
+                   'name')
+        self._test('function f() { let a="data-name".replace(new RegExp("^.+-"), ""); return a; }',
+                   'name')
+        self._test('function f() { let a="data-name".replace(/^.+-/, ""); return a; }',
+                   'name')
+        self._test('function f() { let a="data-name".replace(/a/g, "o"); return a; }',
+                   'doto-nome')
+        self._test('function f() { let a="data-name".replaceAll("a", "o"); return a; }',
+                   'doto-nome')
 
     def test_char_code_at(self):
-        jsi = JSInterpreter('function x(i){return "test".charCodeAt(i)}')
-        self.assertEqual(jsi.call_function('x', 0), 116)
-        self.assertEqual(jsi.call_function('x', 1), 101)
-        self.assertEqual(jsi.call_function('x', 2), 115)
-        self.assertEqual(jsi.call_function('x', 3), 116)
-        self.assertEqual(jsi.call_function('x', 4), None)
-        self.assertEqual(jsi.call_function('x', 'not_a_number'), 116)
+        jsi = JSInterpreter('function f(i){return "test".charCodeAt(i)}')
+        self._test(jsi, 116, args=[0])
+        self._test(jsi, 101, args=[1])
+        self._test(jsi, 115, args=[2])
+        self._test(jsi, 116, args=[3])
+        self._test(jsi, None, args=[4])
+        self._test(jsi, 116, args=['not_a_number'])
 
     def test_bitwise_operators_overflow(self):
-        jsi = JSInterpreter('function x(){return -524999584 << 5}')
-        self.assertEqual(jsi.call_function('x'), 379882496)
+        self._test('function f(){return -524999584 << 5}', 379882496)
+        self._test('function f(){return 1236566549 << 5}', 915423904)
 
-        jsi = JSInterpreter('function x(){return 1236566549 << 5}')
-        self.assertEqual(jsi.call_function('x'), 915423904)
+    def test_bitwise_operators_typecast(self):
+        # madness
+        self._test('function f(){return null << 5}', 0)
+        self._test('function f(){return undefined >> 5}', 0)
+        self._test('function f(){return 42 << NaN}', 42)
+        self._test('function f(){return 42 << Infinity}', 42)
 
-    def test_bitwise_operators_madness(self):
-        jsi = JSInterpreter('function x(){return null << 5}')
-        self.assertEqual(jsi.call_function('x'), 0)
-
-        jsi = JSInterpreter('function x(){return undefined >> 5}')
-        self.assertEqual(jsi.call_function('x'), 0)
-
-        jsi = JSInterpreter('function x(){return 42 << NaN}')
-        self.assertEqual(jsi.call_function('x'), 42)
-
-        jsi = JSInterpreter('function x(){return 42 << Infinity}')
-        self.assertEqual(jsi.call_function('x'), 42)
+    def test_negative(self):
+        self._test('function f(){return 2    *    -2.0    ;}', -4)
+        self._test('function f(){return 2    -    - -2    ;}', 0)
+        self._test('function f(){return 2    -    - - -2  ;}', 4)
+        self._test('function f(){return 2    -    + + - -2;}', 0)
+        self._test('function f(){return 2    +    - + - -2;}', 0)
 
     def test_32066(self):
-        jsi = JSInterpreter("function x(){return Math.pow(3, 5) + new Date('1970-01-01T08:01:42.000+08:00') / 1000 * -239 - -24205;}")
-        self.assertEqual(jsi.call_function('x'), 70)
+        self._test(
+            "function f(){return Math.pow(3, 5) + new Date('1970-01-01T08:01:42.000+08:00') / 1000 * -239 - -24205;}",
+            70)
 
-    def test_unary_operators(self):
-        jsi = JSInterpreter('function f(){return 2  -  - - 2;}')
-        self.assertEqual(jsi.call_function('f'), 0)
-        jsi = JSInterpreter('function f(){return 2 + - + - - 2;}')
-        self.assertEqual(jsi.call_function('f'), 0)
-        # https://github.com/ytdl-org/youtube-dl/issues/32815
-        jsi = JSInterpreter('function f(){return 0  - 7 * - 6;}')
-        self.assertEqual(jsi.call_function('f'), 42)
-
-    """ # fails so far
+    @unittest.skip('Not yet working')
     def test_packed(self):
-        jsi = JSInterpreter('''function x(p,a,c,k,e,d){while(c--)if(k[c])p=p.replace(new RegExp('\\b'+c.toString(a)+'\\b','g'),k[c]);return p}''')
-        self.assertEqual(jsi.call_function('x', '''h 7=g("1j");7.7h({7g:[{33:"w://7f-7e-7d-7c.v.7b/7a/79/78/77/76.74?t=73&s=2s&e=72&f=2t&71=70.0.0.1&6z=6y&6x=6w"}],6v:"w://32.v.u/6u.31",16:"r%",15:"r%",6t:"6s",6r:"",6q:"l",6p:"l",6o:"6n",6m:\'6l\',6k:"6j",9:[{33:"/2u?b=6i&n=50&6h=w://32.v.u/6g.31",6f:"6e"}],1y:{6d:1,6c:\'#6b\',6a:\'#69\',68:"67",66:30,65:r,},"64":{63:"%62 2m%m%61%5z%5y%5x.u%5w%5v%5u.2y%22 2k%m%1o%22 5t%m%1o%22 5s%m%1o%22 2j%m%5r%22 16%m%5q%22 15%m%5p%22 5o%2z%5n%5m%2z",5l:"w://v.u/d/1k/5k.2y",5j:[]},\'5i\':{"5h":"5g"},5f:"5e",5d:"w://v.u",5c:{},5b:l,1x:[0.25,0.50,0.75,1,1.25,1.5,2]});h 1m,1n,5a;h 59=0,58=0;h 7=g("1j");h 2x=0,57=0,56=0;$.55({54:{\'53-52\':\'2i-51\'}});7.j(\'4z\',6(x){c(5>0&&x.1l>=5&&1n!=1){1n=1;$(\'q.4y\').4x(\'4w\')}});7.j(\'13\',6(x){2x=x.1l});7.j(\'2g\',6(x){2w(x)});7.j(\'4v\',6(){$(\'q.2v\').4u()});6 2w(x){$(\'q.2v\').4t();c(1m)19;1m=1;17=0;c(4s.4r===l){17=1}$.4q(\'/2u?b=4p&2l=1k&4o=2t-4n-4m-2s-4l&4k=&4j=&4i=&17=\'+17,6(2r){$(\'#4h\').4g(2r)});$(\'.3-8-4f-4e:4d("4c")\').2h(6(e){2q();g().4b(0);g().4a(l)});6 2q(){h $14=$("<q />").2p({1l:"49",16:"r%",15:"r%",48:0,2n:0,2o:47,46:"45(10%, 10%, 10%, 0.4)","44-43":"42"});$("<41 />").2p({16:"60%",15:"60%",2o:40,"3z-2n":"3y"}).3x({\'2m\':\'/?b=3w&2l=1k\',\'2k\':\'0\',\'2j\':\'2i\'}).2f($14);$14.2h(6(){$(3v).3u();g().2g()});$14.2f($(\'#1j\'))}g().13(0);}6 3t(){h 9=7.1b(2e);2d.2c(9);c(9.n>1){1r(i=0;i<9.n;i++){c(9[i].1a==2e){2d.2c(\'!!=\'+i);7.1p(i)}}}}7.j(\'3s\',6(){g().1h("/2a/3r.29","3q 10 28",6(){g().13(g().27()+10)},"2b");$("q[26=2b]").23().21(\'.3-20-1z\');g().1h("/2a/3p.29","3o 10 28",6(){h 12=g().27()-10;c(12<0)12=0;g().13(12)},"24");$("q[26=24]").23().21(\'.3-20-1z\');});6 1i(){}7.j(\'3n\',6(){1i()});7.j(\'3m\',6(){1i()});7.j("k",6(y){h 9=7.1b();c(9.n<2)19;$(\'.3-8-3l-3k\').3j(6(){$(\'#3-8-a-k\').1e(\'3-8-a-z\');$(\'.3-a-k\').p(\'o-1f\',\'11\')});7.1h("/3i/3h.3g","3f 3e",6(){$(\'.3-1w\').3d(\'3-8-1v\');$(\'.3-8-1y, .3-8-1x\').p(\'o-1g\',\'11\');c($(\'.3-1w\').3c(\'3-8-1v\')){$(\'.3-a-k\').p(\'o-1g\',\'l\');$(\'.3-a-k\').p(\'o-1f\',\'l\');$(\'.3-8-a\').1e(\'3-8-a-z\');$(\'.3-8-a:1u\').3b(\'3-8-a-z\')}3a{$(\'.3-a-k\').p(\'o-1g\',\'11\');$(\'.3-a-k\').p(\'o-1f\',\'11\');$(\'.3-8-a:1u\').1e(\'3-8-a-z\')}},"39");7.j("38",6(y){1d.37(\'1c\',y.9[y.36].1a)});c(1d.1t(\'1c\')){35("1s(1d.1t(\'1c\'));",34)}});h 18;6 1s(1q){h 9=7.1b();c(9.n>1){1r(i=0;i<9.n;i++){c(9[i].1a==1q){c(i==18){19}18=i;7.1p(i)}}}}',36,270,'|||jw|||function|player|settings|tracks|submenu||if||||jwplayer|var||on|audioTracks|true|3D|length|aria|attr|div|100|||sx|filemoon|https||event|active||false|tt|seek|dd|height|width|adb|current_audio|return|name|getAudioTracks|default_audio|localStorage|removeClass|expanded|checked|addButton|callMeMaybe|vplayer|0fxcyc2ajhp1|position|vvplay|vvad|220|setCurrentAudioTrack|audio_name|for|audio_set|getItem|last|open|controls|playbackRates|captions|rewind|icon|insertAfter||detach|ff00||button|getPosition|sec|png|player8|ff11|log|console|track_name|appendTo|play|click|no|scrolling|frameborder|file_code|src|top|zIndex|css|showCCform|data|1662367683|383371|dl|video_ad|doPlay|prevt|mp4|3E||jpg|thumbs|file|300|setTimeout|currentTrack|setItem|audioTrackChanged|dualSound|else|addClass|hasClass|toggleClass|Track|Audio|svg|dualy|images|mousedown|buttons|topbar|playAttemptFailed|beforePlay|Rewind|fr|Forward|ff|ready|set_audio_track|remove|this|upload_srt|prop|50px|margin|1000001|iframe|center|align|text|rgba|background|1000000|left|absolute|pause|setCurrentCaptions|Upload|contains|item|content|html|fviews|referer|prem|embed|3e57249ef633e0d03bf76ceb8d8a4b65|216|83|hash|view|get|TokenZir|window|hide|show|complete|slow|fadeIn|video_ad_fadein|time||cache|Cache|Content|headers|ajaxSetup|v2done|tott|vastdone2|vastdone1|vvbefore|playbackRateControls|cast|aboutlink|FileMoon|abouttext|UHD|1870|qualityLabels|sites|GNOME_POWER|link|2Fiframe|3C|allowfullscreen|22360|22640|22no|marginheight|marginwidth|2FGNOME_POWER|2F0fxcyc2ajhp1|2Fe|2Ffilemoon|2F|3A||22https|3Ciframe|code|sharing|fontOpacity|backgroundOpacity|Tahoma|fontFamily|303030|backgroundColor|FFFFFF|color|userFontScale|thumbnails|kind|0fxcyc2ajhp10000|url|get_slides|start|startparam|none|preload|html5|primary|hlshtml|androidhls|duration|uniform|stretching|0fxcyc2ajhp1_xt|image|2048|sp|6871|asn|127|srv|43200|_g3XlBcu2lmD9oDexD2NLWSmah2Nu3XcDrl93m9PwXY|m3u8||master|0fxcyc2ajhp1_x|00076|01|hls2|to|s01|delivery|storage|moon|sources|setup'''.split('|')))
-    """
+        self._test(
+            '''function f(p,a,c,k,e,d){while(c--)if(k[c])p=p.replace(new RegExp('\\b'+c.toString(a)+'\\b','g'),k[c]);return p}''',
+            '''h 7=g("1j");7.7h({7g:[{33:"w://7f-7e-7d-7c.v.7b/7a/79/78/77/76.74?t=73&s=2s&e=72&f=2t&71=70.0.0.1&6z=6y&6x=6w"}],6v:"w://32.v.u/6u.31",16:"r%",15:"r%",6t:"6s",6r:"",6q:"l",6p:"l",6o:"6n",6m:\'6l\',6k:"6j",9:[{33:"/2u?b=6i&n=50&6h=w://32.v.u/6g.31",6f:"6e"}],1y:{6d:1,6c:\'#6b\',6a:\'#69\',68:"67",66:30,65:r,},"64":{63:"%62 2m%m%61%5z%5y%5x.u%5w%5v%5u.2y%22 2k%m%1o%22 5t%m%1o%22 5s%m%1o%22 2j%m%5r%22 16%m%5q%22 15%m%5p%22 5o%2z%5n%5m%2z",5l:"w://v.u/d/1k/5k.2y",5j:[]},\'5i\':{"5h":"5g"},5f:"5e",5d:"w://v.u",5c:{},5b:l,1x:[0.25,0.50,0.75,1,1.25,1.5,2]});h 1m,1n,5a;h 59=0,58=0;h 7=g("1j");h 2x=0,57=0,56=0;$.55({54:{\'53-52\':\'2i-51\'}});7.j(\'4z\',6(x){c(5>0&&x.1l>=5&&1n!=1){1n=1;$(\'q.4y\').4x(\'4w\')}});7.j(\'13\',6(x){2x=x.1l});7.j(\'2g\',6(x){2w(x)});7.j(\'4v\',6(){$(\'q.2v\').4u()});6 2w(x){$(\'q.2v\').4t();c(1m)19;1m=1;17=0;c(4s.4r===l){17=1}$.4q(\'/2u?b=4p&2l=1k&4o=2t-4n-4m-2s-4l&4k=&4j=&4i=&17=\'+17,6(2r){$(\'#4h\').4g(2r)});$(\'.3-8-4f-4e:4d("4c")\').2h(6(e){2q();g().4b(0);g().4a(l)});6 2q(){h $14=$("<q />").2p({1l:"49",16:"r%",15:"r%",48:0,2n:0,2o:47,46:"45(10%, 10%, 10%, 0.4)","44-43":"42"});$("<41 />").2p({16:"60%",15:"60%",2o:40,"3z-2n":"3y"}).3x({\'2m\':\'/?b=3w&2l=1k\',\'2k\':\'0\',\'2j\':\'2i\'}).2f($14);$14.2h(6(){$(3v).3u();g().2g()});$14.2f($(\'#1j\'))}g().13(0);}6 3t(){h 9=7.1b(2e);2d.2c(9);c(9.n>1){1r(i=0;i<9.n;i++){c(9[i].1a==2e){2d.2c(\'!!=\'+i);7.1p(i)}}}}7.j(\'3s\',6(){g().1h("/2a/3r.29","3q 10 28",6(){g().13(g().27()+10)},"2b");$("q[26=2b]").23().21(\'.3-20-1z\');g().1h("/2a/3p.29","3o 10 28",6(){h 12=g().27()-10;c(12<0)12=0;g().13(12)},"24");$("q[26=24]").23().21(\'.3-20-1z\');});6 1i(){}7.j(\'3n\',6(){1i()});7.j(\'3m\',6(){1i()});7.j("k",6(y){h 9=7.1b();c(9.n<2)19;$(\'.3-8-3l-3k\').3j(6(){$(\'#3-8-a-k\').1e(\'3-8-a-z\');$(\'.3-a-k\').p(\'o-1f\',\'11\')});7.1h("/3i/3h.3g","3f 3e",6(){$(\'.3-1w\').3d(\'3-8-1v\');$(\'.3-8-1y, .3-8-1x\').p(\'o-1g\',\'11\');c($(\'.3-1w\').3c(\'3-8-1v\')){$(\'.3-a-k\').p(\'o-1g\',\'l\');$(\'.3-a-k\').p(\'o-1f\',\'l\');$(\'.3-8-a\').1e(\'3-8-a-z\');$(\'.3-8-a:1u\').3b(\'3-8-a-z\')}3a{$(\'.3-a-k\').p(\'o-1g\',\'11\');$(\'.3-a-k\').p(\'o-1f\',\'11\');$(\'.3-8-a:1u\').1e(\'3-8-a-z\')}},"39");7.j("38",6(y){1d.37(\'1c\',y.9[y.36].1a)});c(1d.1t(\'1c\')){35("1s(1d.1t(\'1c\'));",34)}});h 18;6 1s(1q){h 9=7.1b();c(9.n>1){1r(i=0;i<9.n;i++){c(9[i].1a==1q){c(i==18){19}18=i;7.1p(i)}}}}',36,270,'|||jw|||function|player|settings|tracks|submenu||if||||jwplayer|var||on|audioTracks|true|3D|length|aria|attr|div|100|||sx|filemoon|https||event|active||false|tt|seek|dd|height|width|adb|current_audio|return|name|getAudioTracks|default_audio|localStorage|removeClass|expanded|checked|addButton|callMeMaybe|vplayer|0fxcyc2ajhp1|position|vvplay|vvad|220|setCurrentAudioTrack|audio_name|for|audio_set|getItem|last|open|controls|playbackRates|captions|rewind|icon|insertAfter||detach|ff00||button|getPosition|sec|png|player8|ff11|log|console|track_name|appendTo|play|click|no|scrolling|frameborder|file_code|src|top|zIndex|css|showCCform|data|1662367683|383371|dl|video_ad|doPlay|prevt|mp4|3E||jpg|thumbs|file|300|setTimeout|currentTrack|setItem|audioTrackChanged|dualSound|else|addClass|hasClass|toggleClass|Track|Audio|svg|dualy|images|mousedown|buttons|topbar|playAttemptFailed|beforePlay|Rewind|fr|Forward|ff|ready|set_audio_track|remove|this|upload_srt|prop|50px|margin|1000001|iframe|center|align|text|rgba|background|1000000|left|absolute|pause|setCurrentCaptions|Upload|contains|item|content|html|fviews|referer|prem|embed|3e57249ef633e0d03bf76ceb8d8a4b65|216|83|hash|view|get|TokenZir|window|hide|show|complete|slow|fadeIn|video_ad_fadein|time||cache|Cache|Content|headers|ajaxSetup|v2done|tott|vastdone2|vastdone1|vvbefore|playbackRateControls|cast|aboutlink|FileMoon|abouttext|UHD|1870|qualityLabels|sites|GNOME_POWER|link|2Fiframe|3C|allowfullscreen|22360|22640|22no|marginheight|marginwidth|2FGNOME_POWER|2F0fxcyc2ajhp1|2Fe|2Ffilemoon|2F|3A||22https|3Ciframe|code|sharing|fontOpacity|backgroundOpacity|Tahoma|fontFamily|303030|backgroundColor|FFFFFF|color|userFontScale|thumbnails|kind|0fxcyc2ajhp10000|url|get_slides|start|startparam|none|preload|html5|primary|hlshtml|androidhls|duration|uniform|stretching|0fxcyc2ajhp1_xt|image|2048|sp|6871|asn|127|srv|43200|_g3XlBcu2lmD9oDexD2NLWSmah2Nu3XcDrl93m9PwXY|m3u8||master|0fxcyc2ajhp1_x|00076|01|hls2|to|s01|delivery|storage|moon|sources|setup'''.split('|'))
 
 
 if __name__ == '__main__':
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 02adf6678..799497acb 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -20,7 +20,9 @@ from .compat import (
     compat_basestring,
     compat_chr,
     compat_collections_chain_map as ChainMap,
+    compat_filter as filter,
     compat_itertools_zip_longest as zip_longest,
+    compat_map as map,
     compat_str,
 )
 
@@ -252,7 +254,7 @@ class Debugger(object):
                     cls.write('=> Raises:', e, '<-|', stmt, level=allow_recursion)
                 raise
             if cls.ENABLED and stmt.strip():
-                if should_ret or not repr(ret) == stmt:
+                if should_ret or repr(ret) != stmt:
                     cls.write(['->', '=>'][should_ret], repr(ret), '<-|', stmt, level=allow_recursion)
             return ret, should_ret
         return interpret_statement
@@ -365,6 +367,8 @@ class JSInterpreter(object):
         start, splits, pos, delim_len = 0, 0, 0, len(delim) - 1
         in_quote, escaping, after_op, in_regex_char_group = None, False, True, False
         skipping = 0
+        if skip_delims:
+            skip_delims = variadic(skip_delims)
         for idx, char in enumerate(expr):
             paren_delta = 0
             if not in_quote:
@@ -391,7 +395,7 @@ class JSInterpreter(object):
                 continue
             elif pos == 0 and skip_delims:
                 here = expr[idx:]
-                for s in variadic(skip_delims):
+                for s in skip_delims:
                     if here.startswith(s) and s:
                         skipping = len(s) - 1
                         break
@@ -412,7 +416,6 @@ class JSInterpreter(object):
         if delim is None:
             delim = expr and _MATCHING_PARENS[expr[0]]
         separated = list(cls._separate(expr, delim, 1))
-
         if len(separated) < 2:
             raise cls.Exception('No terminating paren {delim} in {expr!r:.5500}'.format(**locals()))
         return separated[0][1:].strip(), separated[1].strip()
@@ -487,6 +490,7 @@ class JSInterpreter(object):
         # fails on (eg) if (...) stmt1; else stmt2;
         sub_statements = list(self._separate(stmt, ';')) or ['']
         expr = stmt = sub_statements.pop().strip()
+
         for sub_stmt in sub_statements:
             ret, should_return = self.interpret_statement(sub_stmt, local_vars, allow_recursion)
             if should_return:
@@ -626,8 +630,7 @@ class JSInterpreter(object):
                     if m.group('err'):
                         catch_vars[m.group('err')] = err.error if isinstance(err, JS_Throw) else err
                     catch_vars = local_vars.new_child(m=catch_vars)
-                    err = None
-                    pending = self.interpret_statement(sub_expr, catch_vars, allow_recursion)
+                    err, pending = None, self.interpret_statement(sub_expr, catch_vars, allow_recursion)
 
             m = self._FINALLY_RE.match(expr)
             if m:
@@ -877,12 +880,12 @@ class JSInterpreter(object):
                     self.interpret_expression(v, local_vars, allow_recursion)
                     for v in self._separate(arg_str)]
 
-                if obj == compat_str:
+                if obj is compat_str:
                     if member == 'fromCharCode':
                         assertion(argvals, 'takes one or more arguments')
                         return ''.join(map(compat_chr, argvals))
                     raise self.Exception('Unsupported string method ' + member, expr=expr)
-                elif obj == float:
+                elif obj is float:
                     if member == 'pow':
                         assertion(len(argvals) == 2, 'takes two arguments')
                         return argvals[0] ** argvals[1]
@@ -907,12 +910,12 @@ class JSInterpreter(object):
                 elif member == 'splice':
                     assertion(isinstance(obj, list), 'must be applied on a list')
                     assertion(argvals, 'takes one or more arguments')
-                    index, howMany = map(int, (argvals + [len(obj)])[:2])
+                    index, how_many = map(int, (argvals + [len(obj)])[:2])
                     if index < 0:
                         index += len(obj)
                     add_items = argvals[2:]
                     res = []
-                    for i in range(index, min(index + howMany, len(obj))):
+                    for _ in range(index, min(index + how_many, len(obj))):
                         res.append(obj.pop(index))
                     for i, item in enumerate(add_items):
                         obj.insert(index + i, item)
@@ -1002,28 +1005,25 @@ class JSInterpreter(object):
     def extract_object(self, objname):
         _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
         obj = {}
-        fields = None
-        for obj_m in re.finditer(
+        fields = next(filter(None, (
+            obj_m.group('fields') for obj_m in re.finditer(
                 r'''(?xs)
                     {0}\s*\.\s*{1}|{1}\s*=\s*\{{\s*
                         (?P<fields>({2}\s*:\s*function\s*\(.*?\)\s*\{{.*?}}(?:,\s*)?)*)
                     }}\s*;
                 '''.format(_NAME_RE, re.escape(objname), _FUNC_NAME_RE),
-                self.code):
-            fields = obj_m.group('fields')
-            if fields:
-                break
-        else:
+                self.code))), None)
+        if not fields:
             raise self.Exception('Could not find object ' + objname)
         # Currently, it only supports function definitions
-        fields_m = re.finditer(
-            r'''(?x)
-                (?P<key>%s)\s*:\s*function\s*\((?P<args>(?:%s|,)*)\){(?P<code>[^}]+)}
-            ''' % (_FUNC_NAME_RE, _NAME_RE),
-            fields)
-        for f in fields_m:
+        for f in re.finditer(
+                r'''(?x)
+                    (?P<key>%s)\s*:\s*function\s*\((?P<args>(?:%s|,)*)\){(?P<code>[^}]+)}
+                ''' % (_FUNC_NAME_RE, _NAME_RE),
+                fields):
             argnames = self.build_arglist(f.group('args'))
-            obj[remove_quotes(f.group('key'))] = self.build_function(argnames, f.group('code'))
+            name = remove_quotes(f.group('key'))
+            obj[name] = function_with_repr(self.build_function(argnames, f.group('code')), 'F<{0}>'.format(name))
 
         return obj
 
@@ -1058,7 +1058,7 @@ class JSInterpreter(object):
     def extract_function(self, funcname):
         return function_with_repr(
             self.extract_function_from_code(*self.extract_function_code(funcname)),
-            'F<%s>' % (funcname, ))
+            'F<%s>' % (funcname,))
 
     def extract_function_from_code(self, argnames, code, *global_stack):
         local_vars = {}
@@ -1067,7 +1067,7 @@ class JSInterpreter(object):
             if mobj is None:
                 break
             start, body_start = mobj.span()
-            body, remaining = self._separate_at_paren(code[body_start - 1:], '}')
+            body, remaining = self._separate_at_paren(code[body_start - 1:])
             name = self._named_object(local_vars, self.extract_function_from_code(
                 [x.strip() for x in mobj.group('args').split(',')],
                 body, local_vars, *global_stack))
@@ -1095,8 +1095,7 @@ class JSInterpreter(object):
         argnames = tuple(argnames)
 
         def resf(args, kwargs={}, allow_recursion=100):
-            global_stack[0].update(
-                zip_longest(argnames, args, fillvalue=None))
+            global_stack[0].update(zip_longest(argnames, args, fillvalue=None))
             global_stack[0].update(kwargs)
             var_stack = LocalNameSpace(*global_stack)
             ret, should_abort = self.interpret_statement(code.replace('\n', ' '), var_stack, allow_recursion - 1)

From 76ac69917ec76ba663da843795f46916831e6da9 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 10 Jul 2024 18:02:11 +0100
Subject: [PATCH 780/860] [jsinterp] Further improve expression parsing (fix
 fd8242e) Passes tests from yt-dlp

---
 youtube_dl/jsinterp.py | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 799497acb..a2074a91e 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -804,16 +804,19 @@ class JSInterpreter(object):
             if op in ('+', '-'):
                 # simplify/adjust consecutive instances of these operators
                 undone = 0
-                while len(separated) > 1 and not separated[-1].strip():
+                separated = [s.strip() for s in separated]
+                while len(separated) > 1 and not separated[-1]:
                     undone += 1
                     separated.pop()
                 if op == '-' and undone % 2 != 0:
                     right_expr = op + right_expr
                 elif op == '+':
-                    while len(separated) > 1 and separated[-1].strip() in self.OP_CHARS:
+                    while len(separated) > 1 and set(separated[-1]) <= self.OP_CHARS:
+                        right_expr = separated.pop() + right_expr
+                    if separated[-1][-1:] in self.OP_CHARS:
                         right_expr = separated.pop() + right_expr
                 # hanging op at end of left => unary + (strip) or - (push right)
-                left_val = separated[-1]
+                left_val = separated[-1] if separated else ''
                 for dm_op in ('*', '%', '/', '**'):
                     bodmas = tuple(self._separate(left_val, dm_op, skip_delims=skip_delim))
                     if len(bodmas) > 1 and not bodmas[-1].strip():

From d35ce6ce95aac9b98c5f8272824a82e4623b777a Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 10 Jul 2024 18:07:20 +0100
Subject: [PATCH 781/860] [jsinterp] Support functionality for player
 `b22ef6e7` * support `prototype` for call() and apply() (yt-dlp/yt-dlp#10392,
 thx Grub4k) * map JS `Array` to `list`

---
 test/test_jsinterp.py  | 27 +++++++++++++++++++++++++++
 youtube_dl/jsinterp.py | 24 +++++++++++++++++++++---
 2 files changed, 48 insertions(+), 3 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 86d7f0d46..104e766be 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -398,6 +398,33 @@ class TestJSInterpreter(unittest.TestCase):
             '''function f(p,a,c,k,e,d){while(c--)if(k[c])p=p.replace(new RegExp('\\b'+c.toString(a)+'\\b','g'),k[c]);return p}''',
             '''h 7=g("1j");7.7h({7g:[{33:"w://7f-7e-7d-7c.v.7b/7a/79/78/77/76.74?t=73&s=2s&e=72&f=2t&71=70.0.0.1&6z=6y&6x=6w"}],6v:"w://32.v.u/6u.31",16:"r%",15:"r%",6t:"6s",6r:"",6q:"l",6p:"l",6o:"6n",6m:\'6l\',6k:"6j",9:[{33:"/2u?b=6i&n=50&6h=w://32.v.u/6g.31",6f:"6e"}],1y:{6d:1,6c:\'#6b\',6a:\'#69\',68:"67",66:30,65:r,},"64":{63:"%62 2m%m%61%5z%5y%5x.u%5w%5v%5u.2y%22 2k%m%1o%22 5t%m%1o%22 5s%m%1o%22 2j%m%5r%22 16%m%5q%22 15%m%5p%22 5o%2z%5n%5m%2z",5l:"w://v.u/d/1k/5k.2y",5j:[]},\'5i\':{"5h":"5g"},5f:"5e",5d:"w://v.u",5c:{},5b:l,1x:[0.25,0.50,0.75,1,1.25,1.5,2]});h 1m,1n,5a;h 59=0,58=0;h 7=g("1j");h 2x=0,57=0,56=0;$.55({54:{\'53-52\':\'2i-51\'}});7.j(\'4z\',6(x){c(5>0&&x.1l>=5&&1n!=1){1n=1;$(\'q.4y\').4x(\'4w\')}});7.j(\'13\',6(x){2x=x.1l});7.j(\'2g\',6(x){2w(x)});7.j(\'4v\',6(){$(\'q.2v\').4u()});6 2w(x){$(\'q.2v\').4t();c(1m)19;1m=1;17=0;c(4s.4r===l){17=1}$.4q(\'/2u?b=4p&2l=1k&4o=2t-4n-4m-2s-4l&4k=&4j=&4i=&17=\'+17,6(2r){$(\'#4h\').4g(2r)});$(\'.3-8-4f-4e:4d("4c")\').2h(6(e){2q();g().4b(0);g().4a(l)});6 2q(){h $14=$("<q />").2p({1l:"49",16:"r%",15:"r%",48:0,2n:0,2o:47,46:"45(10%, 10%, 10%, 0.4)","44-43":"42"});$("<41 />").2p({16:"60%",15:"60%",2o:40,"3z-2n":"3y"}).3x({\'2m\':\'/?b=3w&2l=1k\',\'2k\':\'0\',\'2j\':\'2i\'}).2f($14);$14.2h(6(){$(3v).3u();g().2g()});$14.2f($(\'#1j\'))}g().13(0);}6 3t(){h 9=7.1b(2e);2d.2c(9);c(9.n>1){1r(i=0;i<9.n;i++){c(9[i].1a==2e){2d.2c(\'!!=\'+i);7.1p(i)}}}}7.j(\'3s\',6(){g().1h("/2a/3r.29","3q 10 28",6(){g().13(g().27()+10)},"2b");$("q[26=2b]").23().21(\'.3-20-1z\');g().1h("/2a/3p.29","3o 10 28",6(){h 12=g().27()-10;c(12<0)12=0;g().13(12)},"24");$("q[26=24]").23().21(\'.3-20-1z\');});6 1i(){}7.j(\'3n\',6(){1i()});7.j(\'3m\',6(){1i()});7.j("k",6(y){h 9=7.1b();c(9.n<2)19;$(\'.3-8-3l-3k\').3j(6(){$(\'#3-8-a-k\').1e(\'3-8-a-z\');$(\'.3-a-k\').p(\'o-1f\',\'11\')});7.1h("/3i/3h.3g","3f 3e",6(){$(\'.3-1w\').3d(\'3-8-1v\');$(\'.3-8-1y, .3-8-1x\').p(\'o-1g\',\'11\');c($(\'.3-1w\').3c(\'3-8-1v\')){$(\'.3-a-k\').p(\'o-1g\',\'l\');$(\'.3-a-k\').p(\'o-1f\',\'l\');$(\'.3-8-a\').1e(\'3-8-a-z\');$(\'.3-8-a:1u\').3b(\'3-8-a-z\')}3a{$(\'.3-a-k\').p(\'o-1g\',\'11\');$(\'.3-a-k\').p(\'o-1f\',\'11\');$(\'.3-8-a:1u\').1e(\'3-8-a-z\')}},"39");7.j("38",6(y){1d.37(\'1c\',y.9[y.36].1a)});c(1d.1t(\'1c\')){35("1s(1d.1t(\'1c\'));",34)}});h 18;6 1s(1q){h 9=7.1b();c(9.n>1){1r(i=0;i<9.n;i++){c(9[i].1a==1q){c(i==18){19}18=i;7.1p(i)}}}}',36,270,'|||jw|||function|player|settings|tracks|submenu||if||||jwplayer|var||on|audioTracks|true|3D|length|aria|attr|div|100|||sx|filemoon|https||event|active||false|tt|seek|dd|height|width|adb|current_audio|return|name|getAudioTracks|default_audio|localStorage|removeClass|expanded|checked|addButton|callMeMaybe|vplayer|0fxcyc2ajhp1|position|vvplay|vvad|220|setCurrentAudioTrack|audio_name|for|audio_set|getItem|last|open|controls|playbackRates|captions|rewind|icon|insertAfter||detach|ff00||button|getPosition|sec|png|player8|ff11|log|console|track_name|appendTo|play|click|no|scrolling|frameborder|file_code|src|top|zIndex|css|showCCform|data|1662367683|383371|dl|video_ad|doPlay|prevt|mp4|3E||jpg|thumbs|file|300|setTimeout|currentTrack|setItem|audioTrackChanged|dualSound|else|addClass|hasClass|toggleClass|Track|Audio|svg|dualy|images|mousedown|buttons|topbar|playAttemptFailed|beforePlay|Rewind|fr|Forward|ff|ready|set_audio_track|remove|this|upload_srt|prop|50px|margin|1000001|iframe|center|align|text|rgba|background|1000000|left|absolute|pause|setCurrentCaptions|Upload|contains|item|content|html|fviews|referer|prem|embed|3e57249ef633e0d03bf76ceb8d8a4b65|216|83|hash|view|get|TokenZir|window|hide|show|complete|slow|fadeIn|video_ad_fadein|time||cache|Cache|Content|headers|ajaxSetup|v2done|tott|vastdone2|vastdone1|vvbefore|playbackRateControls|cast|aboutlink|FileMoon|abouttext|UHD|1870|qualityLabels|sites|GNOME_POWER|link|2Fiframe|3C|allowfullscreen|22360|22640|22no|marginheight|marginwidth|2FGNOME_POWER|2F0fxcyc2ajhp1|2Fe|2Ffilemoon|2F|3A||22https|3Ciframe|code|sharing|fontOpacity|backgroundOpacity|Tahoma|fontFamily|303030|backgroundColor|FFFFFF|color|userFontScale|thumbnails|kind|0fxcyc2ajhp10000|url|get_slides|start|startparam|none|preload|html5|primary|hlshtml|androidhls|duration|uniform|stretching|0fxcyc2ajhp1_xt|image|2048|sp|6871|asn|127|srv|43200|_g3XlBcu2lmD9oDexD2NLWSmah2Nu3XcDrl93m9PwXY|m3u8||master|0fxcyc2ajhp1_x|00076|01|hls2|to|s01|delivery|storage|moon|sources|setup'''.split('|'))
 
+    def test_join(self):
+        test_input = list('test')
+        tests = [
+            'function f(a, b){return a.join(b)}',
+            'function f(a, b){return Array.prototype.join.call(a, b)}',
+            'function f(a, b){return Array.prototype.join.apply(a, [b])}',
+        ]
+        for test in tests:
+            jsi = JSInterpreter(test)
+            self._test(jsi, 'test', args=[test_input, ''])
+            self._test(jsi, 't-e-s-t', args=[test_input, '-'])
+            self._test(jsi, '', args=[[], '-'])
+
+    def test_split(self):
+        test_result = list('test')
+        tests = [
+            'function f(a, b){return a.split(b)}',
+            'function f(a, b){return String.prototype.split.call(a, b)}',
+            'function f(a, b){return String.prototype.split.apply(a, [b])}',
+        ]
+        for test in tests:
+            jsi = JSInterpreter(test)
+            self._test(jsi, test_result, args=['test', ''])
+            self._test(jsi, test_result, args=['t-e-s-t', '-'])
+            self._test(jsi, [''], args=['', '-'])
+            self._test(jsi, [], args=['', ''])
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index a2074a91e..949f77775 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -850,7 +850,7 @@ class JSInterpreter(object):
                     memb = member
                     raise self.Exception('{memb} {msg}'.format(**locals()), expr=expr)
 
-            def eval_method():
+            def eval_method(variable, member):
                 if (variable, member) == ('console', 'debug'):
                     if Debugger.ENABLED:
                         Debugger.write(self.interpret_expression('[{}]'.format(arg_str), local_vars, allow_recursion))
@@ -858,6 +858,7 @@ class JSInterpreter(object):
                 types = {
                     'String': compat_str,
                     'Math': float,
+                    'Array': list,
                 }
                 obj = local_vars.get(variable)
                 if obj in (JS_Undefined, None):
@@ -883,6 +884,23 @@ class JSInterpreter(object):
                     self.interpret_expression(v, local_vars, allow_recursion)
                     for v in self._separate(arg_str)]
 
+                # Fixup prototype call
+                if isinstance(obj, type):
+                    new_member, rest = member.partition('.')[0::2]
+                    if new_member == 'prototype':
+                        new_member, func_prototype = rest.partition('.')[0::2]
+                        assertion(argvals, 'takes one or more arguments')
+                        assertion(isinstance(argvals[0], obj), 'must bind to type {0}'.format(obj))
+                        if func_prototype == 'call':
+                            obj = argvals.pop(0)
+                        elif func_prototype == 'apply':
+                            assertion(len(argvals) == 2, 'takes two arguments')
+                            obj, argvals = argvals
+                            assertion(isinstance(argvals, list), 'second argument must be a list')
+                        else:
+                            raise self.Exception('Unsupported Function method ' + func_prototype, expr)
+                        member = new_member
+
                 if obj is compat_str:
                     if member == 'fromCharCode':
                         assertion(argvals, 'takes one or more arguments')
@@ -976,11 +994,11 @@ class JSInterpreter(object):
 
             if remaining:
                 ret, should_abort = self.interpret_statement(
-                    self._named_object(local_vars, eval_method()) + remaining,
+                    self._named_object(local_vars, eval_method(variable, member)) + remaining,
                     local_vars, allow_recursion)
                 return ret, should_return or should_abort
             else:
-                return eval_method(), should_return
+                return eval_method(variable, member), should_return
 
         elif md.get('function'):
             fname = m.group('fname')

From 16f5bbc464602773e61eeafef51d1dbc47987bb4 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 10 Jul 2024 18:20:59 +0100
Subject: [PATCH 782/860] [YouTube] Fix nsig processing for player `b22ef6e7` *
 improve extraction of function name (like yt-dlp/yt-dlp#10390) * always use
 JSInterp to extract function code (yt-dlp/yt-dlp#10396, thx seproDev,
 pukkandan)

---
 test/test_youtube_signature.py  |  4 ++++
 youtube_dl/extractor/youtube.py | 24 ++++++++++--------------
 2 files changed, 14 insertions(+), 14 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index cafba7a5c..cc18d0f7b 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -162,6 +162,10 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/590f65a6/player_ias.vflset/en_US/base.js',
         '1tm7-g_A9zsI8_Lay_', 'xI4Vem4Put_rOg',
     ),
+    (
+        'https://www.youtube.com/s/player/b22ef6e7/player_ias.vflset/en_US/base.js',
+        'b6HcntHGkvBLk_FRf', 'kNPW6A7FyP2l8A',
+    ),
 ]
 
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 90c16e172..2e31a8979 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1636,7 +1636,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         try:
             jsi, player_id, func_code = self._extract_n_function_code(video_id, player_url)
         except ExtractorError as e:
-            raise ExtractorError('Unable to extract nsig jsi, player_id, func_codefunction code', cause=e)
+            raise ExtractorError('Unable to extract nsig function code', cause=e)
         if self.get_param('youtube_print_sig_code'):
             self.to_screen('Extracted nsig function from {0}:\n{1}\n'.format(
                 player_id, func_code[1]))
@@ -1658,8 +1658,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     def _extract_n_function_name(self, jscode):
         func_name, idx = self._search_regex(
-            r'\.get\("n"\)\)&&\(b=(?P<nfunc>[a-zA-Z_$][\w$]*)(?:\[(?P<idx>\d+)\])?\([\w$]+\)',
-            jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
+            # new: (b=String.fromCharCode(110),c=a.get(b))&&c=nfunc[idx](c)
+            # old: .get("n"))&&(b=nfunc[idx](b)
+            # older: .get("n"))&&(b=nfunc(b)
+            r'''(?x)
+                (?:\(\s*(?P<b>[a-z])\s*=\s*String\s*\.\s*fromCharCode\s*\(\s*110\s*\)\s*,(?P<c>[a-z])\s*=\s*[a-z]\s*)?
+                \.\s*get\s*\(\s*(?(b)(?P=b)|"n")(?:\s*\)){2}\s*&&\s*\(\s*(?(c)(?P=c)|b)\s*=\s*
+                (?P<nfunc>[a-zA-Z_$][\w$]*)(?:\s*\[(?P<idx>\d+)\])?\s*\(\s*[\w$]+\s*\)
+            ''', jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
         if not idx:
             return func_name
 
@@ -1679,17 +1685,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         func_name = self._extract_n_function_name(jscode)
 
-        # For redundancy
-        func_code = self._search_regex(
-            r'''(?xs)%s\s*=\s*function\s*\((?P<var>[\w$]+)\)\s*
-                     # NB: The end of the regex is intentionally kept strict
-                     {(?P<code>.+?}\s*return\ [\w$]+.join\(""\))};''' % func_name,
-            jscode, 'nsig function', group=('var', 'code'), default=None)
-        if func_code:
-            func_code = ([func_code[0]], func_code[1])
-        else:
-            self.write_debug('Extracting nsig function with jsinterp')
-            func_code = jsi.extract_function_code(func_name)
+        func_code = jsi.extract_function_code(func_name)
 
         self.cache.store('youtube-nsig', player_id, func_code)
         return jsi, player_id, func_code

From 451046d62a1b57cc473625c728ff89d7a66933c3 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 24 Jul 2024 14:33:34 +0100
Subject: [PATCH 783/860] [Youtube] Make n-sig throttling diagnostic up-to-date

---
 youtube_dl/extractor/youtube.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 2e31a8979..bb9f48e4c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1647,7 +1647,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         except JSInterpreter.Exception as e:
             self.report_warning(
                 '%s (%s %s)' % (
-                    'Unable to decode n-parameter: download likely to be throttled',
+                    'Unable to decode n-parameter: expect download to be blocked or throttled',
                     error_to_compat_str(e),
                     traceback.format_exc()),
                 video_id=video_id)
@@ -1670,7 +1670,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             return func_name
 
         return self._parse_json(self._search_regex(
-            r'var {0}\s*=\s*(\[.+?\])\s*[,;]'.format(re.escape(func_name)), jscode,
+            r'var\s+{0}\s*=\s*(\[.+?\])\s*[,;]'.format(re.escape(func_name)), jscode,
             'Initial JS player n function list ({0}.{1})'.format(func_name, idx)),
             func_name, transform_source=js_to_json)[int(idx)]
 

From e1b3fa242cb94eb9dc949ab3f2cace91c46f11bf Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 24 Jul 2024 14:45:52 +0100
Subject: [PATCH 784/860] [Youtube] Find `n` function name in player `3400486c`

Fixes #32877
---
 test/test_youtube_signature.py  | 8 ++++++++
 youtube_dl/extractor/youtube.py | 8 ++++++--
 2 files changed, 14 insertions(+), 2 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index cc18d0f7b..5b4aa3aa0 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -166,6 +166,14 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/b22ef6e7/player_ias.vflset/en_US/base.js',
         'b6HcntHGkvBLk_FRf', 'kNPW6A7FyP2l8A',
     ),
+    (
+        'https://www.youtube.com/s/player/3400486c/player_ias.vflset/en_US/base.js',
+        'lL46g3XifCKUZn1Xfw', 'z767lhet6V2Skl',
+    ),
+    (
+        'https://www.youtube.com/s/player/5604538d/player_ias.vflset/en_US/base.js',
+        '7X-he4jjvMx7BCX', 'sViSydX8IHtdWA',
+    ),
 ]
 
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index bb9f48e4c..84371ff06 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1659,11 +1659,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
     def _extract_n_function_name(self, jscode):
         func_name, idx = self._search_regex(
             # new: (b=String.fromCharCode(110),c=a.get(b))&&c=nfunc[idx](c)
+            # or:  (b="nn"[+a.D],c=a.get(b))&&(c=nfunc[idx](c)s
             # old: .get("n"))&&(b=nfunc[idx](b)
             # older: .get("n"))&&(b=nfunc(b)
             r'''(?x)
-                (?:\(\s*(?P<b>[a-z])\s*=\s*String\s*\.\s*fromCharCode\s*\(\s*110\s*\)\s*,(?P<c>[a-z])\s*=\s*[a-z]\s*)?
-                \.\s*get\s*\(\s*(?(b)(?P=b)|"n")(?:\s*\)){2}\s*&&\s*\(\s*(?(c)(?P=c)|b)\s*=\s*
+                (?:\(\s*(?P<b>[a-z])\s*=\s*(?:
+                    String\s*\.\s*fromCharCode\s*\(\s*110\s*\)|
+                    "n+"\[\s*\+?s*[\w$.]+\s*]
+                )\s*,(?P<c>[a-z])\s*=\s*[a-z]\s*)?
+                \.\s*get\s*\(\s*(?(b)(?P=b)|"n{1,2}")(?:\s*\)){2}\s*&&\s*\(\s*(?(c)(?P=c)|b)\s*=\s*
                 (?P<nfunc>[a-zA-Z_$][\w$]*)(?:\s*\[(?P<idx>\d+)\])?\s*\(\s*[\w$]+\s*\)
             ''', jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
         if not idx:

From 71223bff39551a11b6959a3de2dd9e2f070f3c4f Mon Sep 17 00:00:00 2001
From: Aiur Adept <151766879+aiur-adept@users.noreply.github.com>
Date: Thu, 1 Aug 2024 14:18:34 -0400
Subject: [PATCH 785/860] [Youtube] Fix nsig extraction for player 20dfca59 
 (#32891)

* dirkf's patch for nsig extraction
* add generic search per  yt-dlp/yt-dlp/pull/10611 - thx bashonly

---------

Co-authored-by: dirkf <fieldhouse@gmx.net>
---
 test/test_youtube_signature.py  |  4 ++++
 youtube_dl/extractor/youtube.py | 32 ++++++++++++++++++++++++++------
 2 files changed, 30 insertions(+), 6 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 5b4aa3aa0..1c5f667f5 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -174,6 +174,10 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/5604538d/player_ias.vflset/en_US/base.js',
         '7X-he4jjvMx7BCX', 'sViSydX8IHtdWA',
     ),
+    (
+        'https://www.youtube.com/s/player/20dfca59/player_ias.vflset/en_US/base.js',
+        '-fLCxedkAk4LUTK2', 'O8kfRq1y1eyHGw',
+    ),
 ]
 
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 84371ff06..509e374a4 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1659,18 +1659,38 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
     def _extract_n_function_name(self, jscode):
         func_name, idx = self._search_regex(
             # new: (b=String.fromCharCode(110),c=a.get(b))&&c=nfunc[idx](c)
-            # or:  (b="nn"[+a.D],c=a.get(b))&&(c=nfunc[idx](c)s
+            # or:  (b="nn"[+a.D],c=a.get(b))&&(c=nfunc[idx](c)
+            # or:  (PL(a),b=a.j.n||null)&&(b=nfunc[idx](b)
             # old: .get("n"))&&(b=nfunc[idx](b)
             # older: .get("n"))&&(b=nfunc(b)
             r'''(?x)
-                (?:\(\s*(?P<b>[a-z])\s*=\s*(?:
+                (?:\((?:[\w$()\s]+,)*?\s*(?P<b>[a-z])\s*=\s*(?:
                     String\s*\.\s*fromCharCode\s*\(\s*110\s*\)|
-                    "n+"\[\s*\+?s*[\w$.]+\s*]
-                )\s*,(?P<c>[a-z])\s*=\s*[a-z]\s*)?
-                \.\s*get\s*\(\s*(?(b)(?P=b)|"n{1,2}")(?:\s*\)){2}\s*&&\s*\(\s*(?(c)(?P=c)|b)\s*=\s*
+                    "n+"\[\s*\+?s*[\w$.]+\s*]|
+                    (?P<b1>(?:[\w$]+\s*\.\s*)+n\b(?:(?!&&).)+\))
+                )\s*
+                    (?(b1)
+                          &&\s*\(\s*(?P=b)|
+                          (?:
+                              ,(?P<c>[a-z])\s*=\s*[a-z]\s*)?
+                              \.\s*get\s*\(\s*(?(b)(?P=b)|"n{1,2}")(?:\s*\)){2}\s*
+                              &&\s*\(\s*(?(c)(?P=c)|(?P=b))
+                          )
+                    )\s*=\s*
                 (?P<nfunc>[a-zA-Z_$][\w$]*)(?:\s*\[(?P<idx>\d+)\])?\s*\(\s*[\w$]+\s*\)
-            ''', jscode, 'Initial JS player n function name', group=('nfunc', 'idx'))
+            ''', jscode, 'Initial JS player n function name', group=('nfunc', 'idx'),
+            default=(None, None))
+        # thx bashonly: yt-dlp/yt-dlp/pull/10611
+        if not func_name:
+            self.report_warning('Falling back to generic n function search')
+            return self._search_regex(
+                r'''(?xs)
+                    (?:(?<=[^\w$])|^)       # instead of \b, which ignores $
+                    (?P<name>(?!\d)[a-zA-Z\d_$]+)\s*=\s*function\((?!\d)[a-zA-Z\d_$]+\)
+                    \s*\{(?:(?!};).)+?["']enhanced_except_
+                ''', jscode, 'Initial JS player n function name', group='name')
         if not idx:
+            self.report_warning('Falling back to generic n function search')
             return func_name
 
         return self._parse_json(self._search_regex(

From dbc08fba83a0e6c5215e8cec97eb4076f85eee5f Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 6 Aug 2024 20:44:30 +0100
Subject: [PATCH 786/860] [jsinterp] Improve slice implementation for player
 b12cc44b

	Partly taken from yt-dlp/yt-dlp#10664, thx seproDev
        Fixes #32896
---
 test/test_jsinterp.py          | 28 ++++++++++++++++++++++++++++
 test/test_youtube_signature.py |  4 ++++
 youtube_dl/jsinterp.py         | 13 ++++++++++---
 3 files changed, 42 insertions(+), 3 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 104e766be..c7a4f2cbf 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -425,6 +425,34 @@ class TestJSInterpreter(unittest.TestCase):
             self._test(jsi, [''], args=['', '-'])
             self._test(jsi, [], args=['', ''])
 
+    def test_slice(self):
+        self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice()}', [0, 1, 2, 3, 4, 5, 6, 7, 8])
+        self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(0)}', [0, 1, 2, 3, 4, 5, 6, 7, 8])
+        self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(5)}', [5, 6, 7, 8])
+        self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(99)}', [])
+        self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(-2)}', [7, 8])
+        self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(-99)}', [0, 1, 2, 3, 4, 5, 6, 7, 8])
+        self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(0, 0)}', [])
+        self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(1, 0)}', [])
+        self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(0, 1)}', [0])
+        self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(3, 6)}', [3, 4, 5])
+        self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(1, -1)}', [1, 2, 3, 4, 5, 6, 7])
+        self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(-1, 1)}', [])
+        self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(-3, -1)}', [6, 7])
+        self._test('function f(){return "012345678".slice()}', '012345678')
+        self._test('function f(){return "012345678".slice(0)}', '012345678')
+        self._test('function f(){return "012345678".slice(5)}', '5678')
+        self._test('function f(){return "012345678".slice(99)}', '')
+        self._test('function f(){return "012345678".slice(-2)}', '78')
+        self._test('function f(){return "012345678".slice(-99)}', '012345678')
+        self._test('function f(){return "012345678".slice(0, 0)}', '')
+        self._test('function f(){return "012345678".slice(1, 0)}', '')
+        self._test('function f(){return "012345678".slice(0, 1)}', '0')
+        self._test('function f(){return "012345678".slice(3, 6)}', '345')
+        self._test('function f(){return "012345678".slice(1, -1)}', '1234567')
+        self._test('function f(){return "012345678".slice(-1, 1)}', '')
+        self._test('function f(){return "012345678".slice(-3, -1)}', '67')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 1c5f667f5..56e92fac5 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -178,6 +178,10 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/20dfca59/player_ias.vflset/en_US/base.js',
         '-fLCxedkAk4LUTK2', 'O8kfRq1y1eyHGw',
     ),
+    (
+        'https://www.youtube.com/s/player/b12cc44b/player_ias.vflset/en_US/base.js',
+        'keLa5R2U00sR9SQK', 'N1OGyujjEwMnLw',
+    ),
 ]
 
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 949f77775..a616ad070 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -925,9 +925,16 @@ class JSInterpreter(object):
                     obj.reverse()
                     return obj
                 elif member == 'slice':
-                    assertion(isinstance(obj, list), 'must be applied on a list')
-                    assertion(len(argvals) == 1, 'takes exactly one argument')
-                    return obj[argvals[0]:]
+                    assertion(isinstance(obj, (list, compat_str)), 'must be applied on a list or string')
+                    # From [1]:
+                    # .slice() - like [:]
+                    # .slice(n) - like [n:] (not [slice(n)]
+                    # .slice(m, n) - like [m:n] or [slice(m, n)]
+                    # [1] https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/slice
+                    assertion(len(argvals) <= 2, 'takes between 0 and 2 arguments')
+                    if len(argvals) < 2:
+                        argvals += (None,)
+                    return obj[slice(*argvals)]
                 elif member == 'splice':
                     assertion(isinstance(obj, list), 'must be applied on a list')
                     assertion(argvals, 'takes one or more arguments')

From c5098961b04ce83f4615f2a846c84f803b072639 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 6 Aug 2024 20:59:09 +0100
Subject: [PATCH 787/860] [Youtube] Rework n function extraction pattern 
 Now also succeeds with player b12cc44b

---
 youtube_dl/extractor/youtube.py | 43 ++++++++++++++++++++-------------
 1 file changed, 26 insertions(+), 17 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 509e374a4..6fe520e9a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1661,23 +1661,33 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             # new: (b=String.fromCharCode(110),c=a.get(b))&&c=nfunc[idx](c)
             # or:  (b="nn"[+a.D],c=a.get(b))&&(c=nfunc[idx](c)
             # or:  (PL(a),b=a.j.n||null)&&(b=nfunc[idx](b)
-            # old: .get("n"))&&(b=nfunc[idx](b)
-            # older: .get("n"))&&(b=nfunc(b)
+            # or:  (b="nn"[+a.D],vL(a),c=a.j[b]||null)&&(c=narray[idx](c),a.set(b,c),narray.length||nfunc("")
+            # old: (b=a.get("n"))&&(b=nfunc[idx](b)(?P<c>[a-z])\s*=\s*[a-z]\s*
+            # older: (b=a.get("n"))&&(b=nfunc(b)
             r'''(?x)
-                (?:\((?:[\w$()\s]+,)*?\s*(?P<b>[a-z])\s*=\s*(?:
-                    String\s*\.\s*fromCharCode\s*\(\s*110\s*\)|
-                    "n+"\[\s*\+?s*[\w$.]+\s*]|
-                    (?P<b1>(?:[\w$]+\s*\.\s*)+n\b(?:(?!&&).)+\))
-                )\s*
-                    (?(b1)
-                          &&\s*\(\s*(?P=b)|
-                          (?:
-                              ,(?P<c>[a-z])\s*=\s*[a-z]\s*)?
-                              \.\s*get\s*\(\s*(?(b)(?P=b)|"n{1,2}")(?:\s*\)){2}\s*
-                              &&\s*\(\s*(?(c)(?P=c)|(?P=b))
-                          )
-                    )\s*=\s*
-                (?P<nfunc>[a-zA-Z_$][\w$]*)(?:\s*\[(?P<idx>\d+)\])?\s*\(\s*[\w$]+\s*\)
+                \((?:[\w$()\s]+,)*?\s*      # (
+                (?P<b>[a-z])\s*=\s*         # b=
+                (?:
+                    (?:                     # expect ,c=a.get(b) (etc)
+                        String\s*\.\s*fromCharCode\s*\(\s*110\s*\)|
+                        "n+"\[\s*\+?s*[\w$.]+\s*]
+                    )\s*(?:,[\w$()\s]+(?=,))*|
+                       (?P<old>[\w$]+)      # a (old[er])
+                   )\s*
+                   (?(old)
+                                            # b.get("n")
+                       (?:\.\s*[\w$]+\s*|\[\s*[\w$]+\s*]\s*)*?
+                       (?:\.\s*n|\[\s*"n"\s*]|\.\s*get\s*\(\s*"n"\s*\))
+                       |                    # ,c=a.get(b)
+                       ,\s*(?P<c>[a-z])\s*=\s*[a-z]\s*
+                       (?:\.\s*[\w$]+\s*|\[\s*[\w$]+\s*]\s*)*?
+                       (?:\[\s*(?P=b)\s*]|\.\s*get\s*\(\s*(?P=b)\s*\))
+                   )
+                                            # interstitial junk
+                   \s*(?:\|\|\s*null\s*)?(?:\)\s*)?&&\s*(?:\(\s*)?
+               (?(c)(?P=c)|(?P=b))\s*=\s*   # [c|b]=
+                                            # nfunc|nfunc[idx]
+                   (?P<nfunc>[a-zA-Z_$][\w$]*)(?:\s*\[(?P<idx>\d+)\])?\s*\(\s*[\w$]+\s*\)
             ''', jscode, 'Initial JS player n function name', group=('nfunc', 'idx'),
             default=(None, None))
         # thx bashonly: yt-dlp/yt-dlp/pull/10611
@@ -1690,7 +1700,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     \s*\{(?:(?!};).)+?["']enhanced_except_
                 ''', jscode, 'Initial JS player n function name', group='name')
         if not idx:
-            self.report_warning('Falling back to generic n function search')
             return func_name
 
         return self._parse_json(self._search_regex(

From f28d7178e4afb58a94994796cd81c22183ca33c2 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 12 Dec 2024 04:46:33 +0000
Subject: [PATCH 788/860] [InfoExtractor] Use kwarg maxsplit for re.split * May
 become kw-only in future Pythons

---
 youtube_dl/extractor/common.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 9b0016d07..78704b557 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -3170,7 +3170,7 @@ class InfoExtractor(object):
                     # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as
                     # of jwplayer.flash.swf
                     rtmp_url_parts = re.split(
-                        r'((?:mp4|mp3|flv):)', source_url, 1)
+                        r'((?:mp4|mp3|flv):)', source_url, maxsplit=1)
                     if len(rtmp_url_parts) == 3:
                         rtmp_url, prefix, play_path = rtmp_url_parts
                         a_format.update({

From 118c6d7a17c61f1a1031e5530ca2854a526834d5 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 7 Dec 2024 03:37:39 +0000
Subject: [PATCH 789/860] [jsinterp] Implement `typeof` operator

---
 test/test_jsinterp.py  |  15 +++-
 youtube_dl/jsinterp.py | 152 ++++++++++++++++++++++++++++-------------
 2 files changed, 117 insertions(+), 50 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index c7a4f2cbf..d063bbd36 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -266,7 +266,20 @@ class TestJSInterpreter(unittest.TestCase):
         self._test('function f() { return (l=[0,1,2,3], function(a, b){return a+b})((l[1], l[2]), l[3]) }', 5)
 
     def test_void(self):
-        self._test('function f() { return void 42; }', None)
+        self._test('function f() { return void 42; }', JS_Undefined)
+
+    def test_typeof(self):
+        self._test('function f() { return typeof undefined; }', 'undefined')
+        self._test('function f() { return typeof NaN; }', 'number')
+        self._test('function f() { return typeof Infinity; }', 'number')
+        self._test('function f() { return typeof true; }', 'boolean')
+        self._test('function f() { return typeof null; }', 'object')
+        self._test('function f() { return typeof "a string"; }', 'string')
+        self._test('function f() { return typeof 42; }', 'number')
+        self._test('function f() { return typeof 42.42; }', 'number')
+        self._test('function f() { var g = function(){}; return typeof g; }', 'function')
+        self._test('function f() { return typeof {key: "value"}; }', 'object')
+        # not yet implemented: Symbol, BigInt
 
     def test_return_function(self):
         jsi = JSInterpreter('''
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index a616ad070..acc68c533 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -5,7 +5,7 @@ import json
 import operator
 import re
 
-from functools import update_wrapper
+from functools import update_wrapper, wraps
 
 from .utils import (
     error_to_compat_str,
@@ -20,9 +20,11 @@ from .compat import (
     compat_basestring,
     compat_chr,
     compat_collections_chain_map as ChainMap,
+    compat_contextlib_suppress,
     compat_filter as filter,
     compat_itertools_zip_longest as zip_longest,
     compat_map as map,
+    compat_numeric_types,
     compat_str,
 )
 
@@ -138,6 +140,38 @@ def _js_ternary(cndn, if_true=True, if_false=False):
     return if_true
 
 
+def _js_unary_op(op):
+
+    @wraps_op(op)
+    def wrapped(_, a):
+        return op(a)
+
+    return wrapped
+
+
+# https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/typeof
+def _js_typeof(expr):
+    with compat_contextlib_suppress(TypeError, KeyError):
+        return {
+            JS_Undefined: 'undefined',
+            _NaN: 'number',
+            _Infinity: 'number',
+            True: 'boolean',
+            False: 'boolean',
+            None: 'object',
+        }[expr]
+    for t, n in (
+        (compat_basestring, 'string'),
+        (compat_numeric_types, 'number'),
+    ):
+        if isinstance(expr, t):
+            return n
+    if callable(expr):
+        return 'function'
+    # TODO: Symbol, BigInt
+    return 'object'
+
+
 # (op, definition) in order of binding priority, tightest first
 # avoid dict to maintain order
 # definition None => Defined in JSInterpreter._operator
@@ -176,6 +210,11 @@ _SC_OPERATORS = (
     ('&&', None),
 )
 
+_UNARY_OPERATORS_X = (
+    ('void', _js_unary_op(lambda _: JS_Undefined)),
+    ('typeof', _js_unary_op(_js_typeof)),
+)
+
 _OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS))
 
 _NAME_RE = r'[a-zA-Z_$][\w$]*'
@@ -242,6 +281,7 @@ class Debugger(object):
 
     @classmethod
     def wrap_interpreter(cls, f):
+        @wraps(f)
         def interpret_statement(self, stmt, local_vars, allow_recursion, *args, **kwargs):
             if cls.ENABLED and stmt.strip():
                 cls.write(stmt, level=allow_recursion)
@@ -347,6 +387,8 @@ class JSInterpreter(object):
     def __op_chars(cls):
         op_chars = set(';,[')
         for op in cls._all_operators():
+            if op[0].isalpha():
+                continue
             op_chars.update(op[0])
         return op_chars
 
@@ -425,7 +467,7 @@ class JSInterpreter(object):
         if not _cached:
             _cached.extend(itertools.chain(
                 # Ref: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_Precedence
-                _SC_OPERATORS, _LOG_OPERATORS, _COMP_OPERATORS, _OPERATORS))
+                _SC_OPERATORS, _LOG_OPERATORS, _COMP_OPERATORS, _OPERATORS, _UNARY_OPERATORS_X))
         return _cached
 
     def _operator(self, op, left_val, right_expr, expr, local_vars, allow_recursion):
@@ -479,6 +521,52 @@ class JSInterpreter(object):
     _FINALLY_RE = re.compile(r'finally\s*\{')
     _SWITCH_RE = re.compile(r'switch\s*\(')
 
+    def handle_operators(self, expr, local_vars, allow_recursion):
+
+        for op, _ in self._all_operators():
+            # hackety: </> have higher priority than <</>>, but don't confuse them
+            skip_delim = (op + op) if op in '<>*?' else None
+            if op == '?':
+                skip_delim = (skip_delim, '?.')
+            separated = list(self._separate(expr, op, skip_delims=skip_delim))
+            if len(separated) < 2:
+                continue
+
+            right_expr = separated.pop()
+            # handle operators that are both unary and binary, minimal BODMAS
+            if op in ('+', '-'):
+                # simplify/adjust consecutive instances of these operators
+                undone = 0
+                separated = [s.strip() for s in separated]
+                while len(separated) > 1 and not separated[-1]:
+                    undone += 1
+                    separated.pop()
+                if op == '-' and undone % 2 != 0:
+                    right_expr = op + right_expr
+                elif op == '+':
+                    while len(separated) > 1 and set(separated[-1]) <= self.OP_CHARS:
+                        right_expr = separated.pop() + right_expr
+                    if separated[-1][-1:] in self.OP_CHARS:
+                        right_expr = separated.pop() + right_expr
+                # hanging op at end of left => unary + (strip) or - (push right)
+                left_val = separated[-1] if separated else ''
+                for dm_op in ('*', '%', '/', '**'):
+                    bodmas = tuple(self._separate(left_val, dm_op, skip_delims=skip_delim))
+                    if len(bodmas) > 1 and not bodmas[-1].strip():
+                        expr = op.join(separated) + op + right_expr
+                        if len(separated) > 1:
+                            separated.pop()
+                            right_expr = op.join((left_val, right_expr))
+                        else:
+                            separated = [op.join((left_val, right_expr))]
+                            right_expr = None
+                        break
+                if right_expr is None:
+                    continue
+
+            left_val = self.interpret_expression(op.join(separated), local_vars, allow_recursion)
+            return self._operator(op, left_val, right_expr, expr, local_vars, allow_recursion), True
+
     @Debugger.wrap_interpreter
     def interpret_statement(self, stmt, local_vars, allow_recursion=100):
         if allow_recursion < 0:
@@ -533,9 +621,15 @@ class JSInterpreter(object):
             else:
                 raise self.Exception('Unsupported object {obj:.100}'.format(**locals()), expr=expr)
 
-        if expr.startswith('void '):
-            left = self.interpret_expression(expr[5:], local_vars, allow_recursion)
-            return None, should_return
+        for op, _ in _UNARY_OPERATORS_X:
+            if not expr.startswith(op):
+                continue
+            operand = expr[len(op):]
+            if not operand or operand[0] != ' ':
+                continue
+            op_result = self.handle_operators(expr, local_vars, allow_recursion)
+            if op_result:
+                return op_result[0], should_return
 
         if expr.startswith('{'):
             inner, outer = self._separate_at_paren(expr)
@@ -582,7 +676,7 @@ class JSInterpreter(object):
                 if_expr, expr = self._separate_at_paren(expr)
             else:
                 # may lose ... else ... because of ll.368-374
-                if_expr, expr = self._separate_at_paren(expr, delim=';')
+                if_expr, expr = self._separate_at_paren(' %s;' % (expr,), delim=';')
             else_expr = None
             m = re.match(r'else\s*(?P<block>\{)?', expr)
             if m:
@@ -790,49 +884,9 @@ class JSInterpreter(object):
             idx = self.interpret_expression(m.group('idx'), local_vars, allow_recursion)
             return self._index(val, idx), should_return
 
-        for op, _ in self._all_operators():
-            # hackety: </> have higher priority than <</>>, but don't confuse them
-            skip_delim = (op + op) if op in '<>*?' else None
-            if op == '?':
-                skip_delim = (skip_delim, '?.')
-            separated = list(self._separate(expr, op, skip_delims=skip_delim))
-            if len(separated) < 2:
-                continue
-
-            right_expr = separated.pop()
-            # handle operators that are both unary and binary, minimal BODMAS
-            if op in ('+', '-'):
-                # simplify/adjust consecutive instances of these operators
-                undone = 0
-                separated = [s.strip() for s in separated]
-                while len(separated) > 1 and not separated[-1]:
-                    undone += 1
-                    separated.pop()
-                if op == '-' and undone % 2 != 0:
-                    right_expr = op + right_expr
-                elif op == '+':
-                    while len(separated) > 1 and set(separated[-1]) <= self.OP_CHARS:
-                        right_expr = separated.pop() + right_expr
-                    if separated[-1][-1:] in self.OP_CHARS:
-                        right_expr = separated.pop() + right_expr
-                # hanging op at end of left => unary + (strip) or - (push right)
-                left_val = separated[-1] if separated else ''
-                for dm_op in ('*', '%', '/', '**'):
-                    bodmas = tuple(self._separate(left_val, dm_op, skip_delims=skip_delim))
-                    if len(bodmas) > 1 and not bodmas[-1].strip():
-                        expr = op.join(separated) + op + right_expr
-                        if len(separated) > 1:
-                            separated.pop()
-                            right_expr = op.join((left_val, right_expr))
-                        else:
-                            separated = [op.join((left_val, right_expr))]
-                            right_expr = None
-                        break
-                if right_expr is None:
-                    continue
-
-            left_val = self.interpret_expression(op.join(separated), local_vars, allow_recursion)
-            return self._operator(op, left_val, right_expr, expr, local_vars, allow_recursion), should_return
+        op_result = self.handle_operators(expr, local_vars, allow_recursion)
+        if op_result:
+            return op_result[0], should_return
 
         if md.get('attribute'):
             variable, member, nullish = m.group('var', 'member', 'nullish')

From c1a03b1ac3a453508b358b6d1bc5d158cc80a0ce Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 12 Dec 2024 00:00:32 +0000
Subject: [PATCH 790/860] [jsinterp] Fix and improve loose and strict equality
 operations * reimplement loose equality according to MDN (eg, 1 == "1") *
 improve strict equality (eg, "abc" === "abc" but 'abc' is not 'abc') * add
 tests for above

---
 test/test_jsinterp.py  | 35 ++++++++++++++++--
 youtube_dl/jsinterp.py | 80 ++++++++++++++++++++++++++++++++++++------
 2 files changed, 101 insertions(+), 14 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index d063bbd36..86137d8e5 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# coding: utf-8
 
 from __future__ import unicode_literals
 
@@ -11,7 +12,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 import math
 import re
 
-from youtube_dl.compat import compat_str
+from youtube_dl.compat import compat_str as str
 from youtube_dl.jsinterp import JS_Undefined, JSInterpreter
 
 NaN = object()
@@ -19,7 +20,7 @@ NaN = object()
 
 class TestJSInterpreter(unittest.TestCase):
     def _test(self, jsi_or_code, expected, func='f', args=()):
-        if isinstance(jsi_or_code, compat_str):
+        if isinstance(jsi_or_code, str):
             jsi_or_code = JSInterpreter(jsi_or_code)
         got = jsi_or_code.call_function(func, *args)
         if expected is NaN:
@@ -89,7 +90,35 @@ class TestJSInterpreter(unittest.TestCase):
         self._test('function f(){return 19 & 21;}', 17)
         self._test('function f(){return 11 >> 2;}', 2)
         self._test('function f(){return []? 2+3: 4;}', 5)
+        # equality
+        self._test('function f(){return 1 == 1}', True)
+        self._test('function f(){return 1 == 1.0}', True)
+        self._test('function f(){return 1 == "1"}', True)
         self._test('function f(){return 1 == 2}', False)
+        self._test('function f(){return 1 != "1"}', False)
+        self._test('function f(){return 1 != 2}', True)
+        self._test('function f(){var x = {a: 1}; var y = x; return x == y}', True)
+        self._test('function f(){var x = {a: 1}; return x == {a: 1}}', False)
+        self._test('function f(){return NaN == NaN}', False)
+        self._test('function f(){return null == undefined}', True)
+        self._test('function f(){return "spam, eggs" == "spam, eggs"}', True)
+        # strict equality
+        self._test('function f(){return 1 === 1}', True)
+        self._test('function f(){return 1 === 1.0}', True)
+        self._test('function f(){return 1 === "1"}', False)
+        self._test('function f(){return 1 === 2}', False)
+        self._test('function f(){var x = {a: 1}; var y = x; return x === y}', True)
+        self._test('function f(){var x = {a: 1}; return x === {a: 1}}', False)
+        self._test('function f(){return NaN === NaN}', False)
+        self._test('function f(){return null === undefined}', False)
+        self._test('function f(){return null === null}', True)
+        self._test('function f(){return undefined === undefined}', True)
+        self._test('function f(){return "uninterned" === "uninterned"}', True)
+        self._test('function f(){return 1 === 1}', True)
+        self._test('function f(){return 1 === "1"}', False)
+        self._test('function f(){return 1 !== 1}', False)
+        self._test('function f(){return 1 !== "1"}', True)
+        # expressions
         self._test('function f(){return 0 && 1 || 2;}', 2)
         self._test('function f(){return 0 ?? 42;}', 0)
         self._test('function f(){return "life, the universe and everything" < 42;}', False)
@@ -296,7 +325,7 @@ class TestJSInterpreter(unittest.TestCase):
     def test_undefined(self):
         self._test('function f() { return undefined === undefined; }', True)
         self._test('function f() { return undefined; }', JS_Undefined)
-        self._test('function f() {return undefined ?? 42; }', 42)
+        self._test('function f() { return undefined ?? 42; }', 42)
         self._test('function f() { let v; return v; }', JS_Undefined)
         self._test('function f() { let v; return v**0; }', 1)
         self._test('function f() { let v; return [v>42, v<=42, v&&42, 42&&v]; }',
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index acc68c533..b7a3ef69a 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -1,3 +1,4 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
 import itertools
@@ -64,6 +65,10 @@ _NaN = float('nan')
 _Infinity = float('inf')
 
 
+class JS_Undefined(object):
+    pass
+
+
 def _js_bit_op(op):
 
     def zeroise(x):
@@ -107,12 +112,69 @@ def _js_exp(a, b):
     return (a or 0) ** b
 
 
-def _js_eq_op(op):
+def _js_to_primitive(v):
+    return (
+        ','.join(map(_js_toString, v)) if isinstance(v, list)
+        else '[object Object]' if isinstance(v, dict)
+        else compat_str(v) if not isinstance(v, (
+            compat_numeric_types, compat_basestring, bool))
+        else v
+    )
+
+
+def _js_toString(v):
+    return (
+        'undefined' if v is JS_Undefined
+        else 'Infinity' if v == _Infinity
+        else 'NaN' if v is _NaN
+        else 'null' if v is None
+        else compat_str(v) if isinstance(v, compat_numeric_types)
+        else _js_to_primitive(v))
+
+
+_nullish = frozenset((None, JS_Undefined))
+
+
+def _js_eq(a, b):
+    # NaN != any
+    if _NaN in (a, b):
+        return False
+    # Object is Object
+    if isinstance(a, type(b)) and isinstance(b, (dict, list)):
+        return operator.is_(a, b)
+    # general case
+    if a == b:
+        return True
+    # null == undefined
+    a_b = set((a, b))
+    if a_b & _nullish:
+        return a_b <= _nullish
+    a, b = _js_to_primitive(a), _js_to_primitive(b)
+    if not isinstance(a, compat_basestring):
+        a, b = b, a
+    # Number to String: convert the string to a number
+    # Conversion failure results in ... false
+    if isinstance(a, compat_basestring):
+        return float_or_none(a) == b
+    return a == b
+
+
+def _js_neq(a, b):
+    return not _js_eq(a, b)
+
+
+def _js_id_op(op):
 
     @wraps_op(op)
     def wrapped(a, b):
-        if set((a, b)) <= set((None, JS_Undefined)):
-            return op(a, a)
+        if _NaN in (a, b):
+            return op(_NaN, None)
+        if not isinstance(a, (compat_basestring, compat_numeric_types)):
+            a, b = b, a
+        # strings are === if ==
+        # why 'a' is not 'a': https://stackoverflow.com/a/1504848
+        if isinstance(a, (compat_basestring, compat_numeric_types)):
+            return a == b if op(0, 0) else a != b
         return op(a, b)
 
     return wrapped
@@ -187,10 +249,10 @@ _OPERATORS = (
 )
 
 _COMP_OPERATORS = (
-    ('===', operator.is_),
-    ('!==', operator.is_not),
-    ('==', _js_eq_op(operator.eq)),
-    ('!=', _js_eq_op(operator.ne)),
+    ('===', _js_id_op(operator.is_)),
+    ('!==', _js_id_op(operator.is_not)),
+    ('==', _js_eq),
+    ('!=', _js_neq),
     ('<=', _js_comp_op(operator.le)),
     ('>=', _js_comp_op(operator.ge)),
     ('<', _js_comp_op(operator.lt)),
@@ -222,10 +284,6 @@ _MATCHING_PARENS = dict(zip(*zip('()', '{}', '[]')))
 _QUOTES = '\'"/'
 
 
-class JS_Undefined(object):
-    pass
-
-
 class JS_Break(ExtractorError):
     def __init__(self):
         ExtractorError.__init__(self, 'Invalid break')

From 81e64cacf2b8c144ec1e9d3258db792f0eb8443e Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 12 Dec 2024 00:15:31 +0000
Subject: [PATCH 791/860] [jsinterp] Support multiple indexing (eg a[1][2]) *
 extend single indexing with improved RE (should probably use/have used
 _separate_at_paren()) * fix some cases that should have given undefined, not
 throwing * standardise RE group names * support length of objects, like {1:
 2, 3: 4, length: 42}

---
 test/test_jsinterp.py  | 10 ++++++++++
 youtube_dl/jsinterp.py | 37 ++++++++++++++++++++++---------------
 2 files changed, 32 insertions(+), 15 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 86137d8e5..d2fca2997 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -366,6 +366,16 @@ class TestJSInterpreter(unittest.TestCase):
         self._test('function f() { let a; return a?.qq; }', JS_Undefined)
         self._test('function f() { let a = {m1: 42, m2: 0 }; return a?.qq; }', JS_Undefined)
 
+    def test_indexing(self):
+        self._test('function f() { return [1, 2, 3, 4][3]}', 4)
+        self._test('function f() { return [1, [2, [3, [4]]]][1][1][1][0]}', 4)
+        self._test('function f() { var o = {1: 2, 3: 4}; return o[3]}', 4)
+        self._test('function f() { var o = {1: 2, 3: 4}; return o["3"]}', 4)
+        self._test('function f() { return [1, [2, {3: [4]}]][1][1]["3"][0]}', 4)
+        self._test('function f() { return [1, 2, 3, 4].length}', 4)
+        self._test('function f() { var o = {1: 2, 3: 4}; return o.length}', JS_Undefined)
+        self._test('function f() { var o = {1: 2, 3: 4}; o["length"] = 42; return o.length}', 42)
+
     def test_regex(self):
         self._test('function f() { let a=/,,[/,913,/](,)}/; }', None)
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index b7a3ef69a..248210ce4 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -549,13 +549,14 @@ class JSInterpreter(object):
         except Exception as e:
             raise self.Exception('Failed to evaluate {left_val!r:.50} {op} {right_val!r:.50}'.format(**locals()), expr, cause=e)
 
-    def _index(self, obj, idx, allow_undefined=False):
-        if idx == 'length':
+    def _index(self, obj, idx, allow_undefined=True):
+        if idx == 'length' and isinstance(obj, list):
             return len(obj)
         try:
-            return obj[int(idx)] if isinstance(obj, list) else obj[idx]
-        except Exception as e:
+            return obj[int(idx)] if isinstance(obj, list) else obj[compat_str(idx)]
+        except (TypeError, KeyError, IndexError) as e:
             if allow_undefined:
+                # when is not allowed?
                 return JS_Undefined
             raise self.Exception('Cannot get index {idx!r:.100}'.format(**locals()), expr=repr(obj), cause=e)
 
@@ -882,13 +883,13 @@ class JSInterpreter(object):
 
         m = re.match(r'''(?x)
             (?P<assign>
-                (?P<out>{_NAME_RE})(?:\[(?P<index>[^\]]+?)\])?\s*
+                (?P<out>{_NAME_RE})(?:\[(?P<out_idx>(?:.+?\]\s*\[)*.+?)\])?\s*
                 (?P<op>{_OPERATOR_RE})?
                 =(?!=)(?P<expr>.*)$
             )|(?P<return>
                 (?!if|return|true|false|null|undefined|NaN|Infinity)(?P<name>{_NAME_RE})$
             )|(?P<indexing>
-                (?P<in>{_NAME_RE})\[(?P<idx>.+)\]$
+                (?P<in>{_NAME_RE})\[(?P<in_idx>(?:.+?\]\s*\[)*.+?)\]$
             )|(?P<attribute>
                 (?P<var>{_NAME_RE})(?:(?P<nullish>\?)?\.(?P<member>[^(]+)|\[(?P<member2>[^\]]+)\])\s*
             )|(?P<function>
@@ -898,19 +899,23 @@ class JSInterpreter(object):
         if md.get('assign'):
             left_val = local_vars.get(m.group('out'))
 
-            if not m.group('index'):
+            if not m.group('out_idx'):
                 local_vars[m.group('out')] = self._operator(
                     m.group('op'), left_val, m.group('expr'), expr, local_vars, allow_recursion)
                 return local_vars[m.group('out')], should_return
             elif left_val in (None, JS_Undefined):
                 raise self.Exception('Cannot index undefined variable ' + m.group('out'), expr=expr)
 
-            idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion)
-            if not isinstance(idx, (int, float)):
-                raise self.Exception('List index %s must be integer' % (idx, ), expr=expr)
-            idx = int(idx)
+            indexes = re.split(r'\]\s*\[', m.group('out_idx'))
+            for i, idx in enumerate(indexes, 1):
+                idx = self.interpret_expression(idx, local_vars, allow_recursion)
+                if i < len(indexes):
+                    left_val = self._index(left_val, idx)
+            if isinstance(idx, float):
+                idx = int(idx)
             left_val[idx] = self._operator(
-                m.group('op'), self._index(left_val, idx), m.group('expr'), expr, local_vars, allow_recursion)
+                m.group('op'), self._index(left_val, idx) if m.group('op') else None,
+                m.group('expr'), expr, local_vars, allow_recursion)
             return left_val[idx], should_return
 
         elif expr.isdigit():
@@ -939,8 +944,10 @@ class JSInterpreter(object):
 
         if md.get('indexing'):
             val = local_vars[m.group('in')]
-            idx = self.interpret_expression(m.group('idx'), local_vars, allow_recursion)
-            return self._index(val, idx), should_return
+            for idx in re.split(r'\]\s*\[', m.group('in_idx')):
+                idx = self.interpret_expression(idx, local_vars, allow_recursion)
+                val = self._index(val, idx)
+            return val, should_return
 
         op_result = self.handle_operators(expr, local_vars, allow_recursion)
         if op_result:
@@ -989,7 +996,7 @@ class JSInterpreter(object):
 
                 # Member access
                 if arg_str is None:
-                    return self._index(obj, member, nullish)
+                    return self._index(obj, member)
 
                 # Function call
                 argvals = [

From 5dee6213edddb5ea00775db6b3b73f8355144485 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 12 Dec 2024 00:26:00 +0000
Subject: [PATCH 792/860] [jsinterp] Fix and improve arithmetic operations *
 addition becomes concat with a string operand * improve handling of edgier
 cases * arithmetic in float like JS (more places need cast to int?) *
 increase test coverage

---
 test/test_jsinterp.py  | 20 ++++++++++++++++
 youtube_dl/jsinterp.py | 53 +++++++++++++++++++++++++++---------------
 2 files changed, 54 insertions(+), 19 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index d2fca2997..52eaf1ed8 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -41,16 +41,27 @@ class TestJSInterpreter(unittest.TestCase):
         self._test('function f(){return 42 + 7;}', 49)
         self._test('function f(){return 42 + undefined;}', NaN)
         self._test('function f(){return 42 + null;}', 42)
+        self._test('function f(){return 1 + "";}', '1')
+        self._test('function f(){return 42 + "7";}', '427')
+        self._test('function f(){return false + true;}', 1)
+        self._test('function f(){return "false" + true;}', 'falsetrue')
+        self._test('function f(){return '
+                   '1 + "2" + [3,4] + {k: 56} + null + undefined + Infinity;}',
+                   '123,4[object Object]nullundefinedInfinity')
 
     def test_sub(self):
         self._test('function f(){return 42 - 7;}', 35)
         self._test('function f(){return 42 - undefined;}', NaN)
         self._test('function f(){return 42 - null;}', 42)
+        self._test('function f(){return 42 - "7";}', 35)
+        self._test('function f(){return 42 - "spam";}', NaN)
 
     def test_mul(self):
         self._test('function f(){return 42 * 7;}', 294)
         self._test('function f(){return 42 * undefined;}', NaN)
         self._test('function f(){return 42 * null;}', 0)
+        self._test('function f(){return 42 * "7";}', 294)
+        self._test('function f(){return 42 * "eggs";}', NaN)
 
     def test_div(self):
         jsi = JSInterpreter('function f(a, b){return a / b;}')
@@ -58,17 +69,26 @@ class TestJSInterpreter(unittest.TestCase):
         self._test(jsi, NaN, args=(JS_Undefined, 1))
         self._test(jsi, float('inf'), args=(2, 0))
         self._test(jsi, 0, args=(0, 3))
+        self._test(jsi, 6, args=(42, 7))
+        self._test(jsi, 0, args=(42, float('inf')))
+        self._test(jsi, 6, args=("42", 7))
+        self._test(jsi, NaN, args=("spam", 7))
 
     def test_mod(self):
         self._test('function f(){return 42 % 7;}', 0)
         self._test('function f(){return 42 % 0;}', NaN)
         self._test('function f(){return 42 % undefined;}', NaN)
+        self._test('function f(){return 42 % "7";}', 0)
+        self._test('function f(){return 42 % "beans";}', NaN)
 
     def test_exp(self):
         self._test('function f(){return 42 ** 2;}', 1764)
         self._test('function f(){return 42 ** undefined;}', NaN)
         self._test('function f(){return 42 ** null;}', 1)
+        self._test('function f(){return undefined ** 0;}', 1)
         self._test('function f(){return undefined ** 42;}', NaN)
+        self._test('function f(){return 42 ** "2";}', 1764)
+        self._test('function f(){return 42 ** "spam";}', NaN)
 
     def test_calc(self):
         self._test('function f(a){return 2*a+1;}', 7, args=[3])
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 248210ce4..73934a6af 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -11,6 +11,7 @@ from functools import update_wrapper, wraps
 from .utils import (
     error_to_compat_str,
     ExtractorError,
+    float_or_none,
     js_to_json,
     remove_quotes,
     unified_timestamp,
@@ -81,35 +82,47 @@ def _js_bit_op(op):
     return wrapped
 
 
-def _js_arith_op(op):
+def _js_arith_op(op, div=False):
 
     @wraps_op(op)
     def wrapped(a, b):
         if JS_Undefined in (a, b):
             return _NaN
-        return op(a or 0, b or 0)
+        # null, "" --> 0
+        a, b = (float_or_none(
+            (x.strip() if isinstance(x, compat_basestring) else x) or 0,
+            default=_NaN) for x in (a, b))
+        if _NaN in (a, b):
+            return _NaN
+        try:
+            return op(a, b)
+        except ZeroDivisionError:
+            return _NaN if not (div and (a or b)) else _Infinity
 
     return wrapped
 
 
-def _js_div(a, b):
-    if JS_Undefined in (a, b) or not (a or b):
-        return _NaN
-    return operator.truediv(a or 0, b) if b else _Infinity
+_js_arith_add = _js_arith_op(operator.add)
 
 
-def _js_mod(a, b):
-    if JS_Undefined in (a, b) or not b:
-        return _NaN
-    return (a or 0) % b
+def _js_add(a, b):
+    if not (isinstance(a, compat_basestring) or isinstance(b, compat_basestring)):
+        return _js_arith_add(a, b)
+    if not isinstance(a, compat_basestring):
+        a = _js_toString(a)
+    elif not isinstance(b, compat_basestring):
+        b = _js_toString(b)
+    return operator.concat(a, b)
+
+
+_js_mod = _js_arith_op(operator.mod)
+__js_exp = _js_arith_op(operator.pow)
 
 
 def _js_exp(a, b):
     if not b:
         return 1  # even 0 ** 0 !!
-    elif JS_Undefined in (a, b):
-        return _NaN
-    return (a or 0) ** b
+    return __js_exp(a, b)
 
 
 def _js_to_primitive(v):
@@ -117,7 +130,7 @@ def _js_to_primitive(v):
         ','.join(map(_js_toString, v)) if isinstance(v, list)
         else '[object Object]' if isinstance(v, dict)
         else compat_str(v) if not isinstance(v, (
-            compat_numeric_types, compat_basestring, bool))
+            compat_numeric_types, compat_basestring))
         else v
     )
 
@@ -128,7 +141,9 @@ def _js_toString(v):
         else 'Infinity' if v == _Infinity
         else 'NaN' if v is _NaN
         else 'null' if v is None
-        else compat_str(v) if isinstance(v, compat_numeric_types)
+        # bool <= int: do this first
+        else ('false', 'true')[v] if isinstance(v, bool)
+        else '{0:.7f}'.format(v).rstrip('.0') if isinstance(v, compat_numeric_types)
         else _js_to_primitive(v))
 
 
@@ -240,11 +255,11 @@ def _js_typeof(expr):
 _OPERATORS = (
     ('>>', _js_bit_op(operator.rshift)),
     ('<<', _js_bit_op(operator.lshift)),
-    ('+', _js_arith_op(operator.add)),
+    ('+', _js_add),
     ('-', _js_arith_op(operator.sub)),
     ('*', _js_arith_op(operator.mul)),
     ('%', _js_mod),
-    ('/', _js_div),
+    ('/', _js_arith_op(operator.truediv, div=True)),
     ('**', _js_exp),
 )
 
@@ -873,7 +888,7 @@ class JSInterpreter(object):
             start, end = m.span()
             sign = m.group('pre_sign') or m.group('post_sign')
             ret = local_vars[var]
-            local_vars[var] += 1 if sign[0] == '+' else -1
+            local_vars[var] = _js_add(ret, 1 if sign[0] == '+' else -1)
             if m.group('pre_sign'):
                 ret = local_vars[var]
             expr = expr[:start] + self._dump(ret, local_vars) + expr[end:]
@@ -1023,7 +1038,7 @@ class JSInterpreter(object):
                 if obj is compat_str:
                     if member == 'fromCharCode':
                         assertion(argvals, 'takes one or more arguments')
-                        return ''.join(map(compat_chr, argvals))
+                        return ''.join(compat_chr(int(n)) for n in argvals)
                     raise self.Exception('Unsupported string method ' + member, expr=expr)
                 elif obj is float:
                     if member == 'pow':

From 94fd7746084d87a43e34b094c5db1325f91ce053 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 12 Dec 2024 04:13:06 +0000
Subject: [PATCH 793/860] [jsinterp] Fix and improve split/join * improve
 split/join edge cases * correctly implement regex split (not like re.split)

---
 test/test_jsinterp.py  | 19 +++++++++++++++++
 youtube_dl/jsinterp.py | 48 +++++++++++++++++++++++++++++++++++++-----
 2 files changed, 62 insertions(+), 5 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 52eaf1ed8..b6e87e9f1 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -483,6 +483,13 @@ class TestJSInterpreter(unittest.TestCase):
             self._test(jsi, 't-e-s-t', args=[test_input, '-'])
             self._test(jsi, '', args=[[], '-'])
 
+        self._test('function f(){return '
+                   '[1, 1.0, "abc", {a: 1}, null, undefined, Infinity, NaN].join()}',
+                   '1,1,abc,[object Object],,,Infinity,NaN')
+        self._test('function f(){return '
+                   '[1, 1.0, "abc", {a: 1}, null, undefined, Infinity, NaN].join("~")}',
+                   '1~1~abc~[object Object]~~~Infinity~NaN')
+
     def test_split(self):
         test_result = list('test')
         tests = [
@@ -496,6 +503,18 @@ class TestJSInterpreter(unittest.TestCase):
             self._test(jsi, test_result, args=['t-e-s-t', '-'])
             self._test(jsi, [''], args=['', '-'])
             self._test(jsi, [], args=['', ''])
+        # RegExp split
+        self._test('function f(){return "test".split(/(?:)/)}',
+                   ['t', 'e', 's', 't'])
+        self._test('function f(){return "t-e-s-t".split(/[es-]+/)}',
+                   ['t', 't'])
+        # from MDN: surrogate pairs aren't handled: case 1 fails
+        # self._test('function f(){return "😄😄".split(/(?:)/)}',
+        #            ['\ud83d', '\ude04', '\ud83d', '\ude04'])
+        # case 2 beats Py3.2: it gets the case 1 result
+        if sys.version_info >= (2, 6) and not ((3, 0) <= sys.version_info < (3, 3)):
+            self._test('function f(){return "😄😄".split(/(?:)/u)}',
+                       ['😄', '😄'])
 
     def test_slice(self):
         self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice()}', [0, 1, 2, 3, 4, 5, 6, 7, 8])
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 73934a6af..bec959946 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -397,6 +397,9 @@ class JSInterpreter(object):
         RE_FLAGS = {
             # special knowledge: Python's re flags are bitmask values, current max 128
             # invent new bitmask values well above that for literal parsing
+            # JS 'u' flag is effectively always set (surrogate pairs aren't seen),
+            # but \u{...} and \p{...} escapes aren't handled); no additional JS 'v'
+            # features are supported
             # TODO: execute matches with these flags (remaining: d, y)
             'd': 1024,  # Generate indices for substring matches
             'g': 2048,  # Global search
@@ -404,6 +407,7 @@ class JSInterpreter(object):
             'm': re.M,  # Multi-line search
             's': re.S,  # Allows . to match newline characters
             'u': re.U,  # Treat a pattern as a sequence of unicode code points
+            'v': re.U,  # Like 'u' with extended character class and \p{} syntax
             'y': 4096,  # Perform a "sticky" search that matches starting at the current position in the target string
         }
 
@@ -1047,13 +1051,47 @@ class JSInterpreter(object):
                     raise self.Exception('Unsupported Math method ' + member, expr=expr)
 
                 if member == 'split':
-                    assertion(argvals, 'takes one or more arguments')
-                    assertion(len(argvals) == 1, 'with limit argument is not implemented')
-                    return obj.split(argvals[0]) if argvals[0] else list(obj)
+                    assertion(len(argvals) <= 2, 'takes at most two arguments')
+                    if len(argvals) > 1:
+                        limit = argvals[1]
+                        assertion(isinstance(limit, int) and limit >= 0, 'integer limit >= 0')
+                        if limit == 0:
+                            return []
+                    else:
+                        limit = 0
+                    if len(argvals) == 0:
+                        argvals = [JS_Undefined]
+                    elif isinstance(argvals[0], self.JS_RegExp):
+                        # avoid re.split(), similar but not enough
+
+                        def where():
+                            for m in argvals[0].finditer(obj):
+                                yield m.span(0)
+                            yield (None, None)
+
+                        def splits(limit=limit):
+                            i = 0
+                            for j, jj in where():
+                                if j == jj == 0:
+                                    continue
+                                if j is None and i >= len(obj):
+                                    break
+                                yield obj[i:j]
+                                if jj is None or limit == 1:
+                                    break
+                                limit -= 1
+                                i = jj
+
+                        return list(splits())
+                    return (
+                        obj.split(argvals[0], limit - 1) if argvals[0] and argvals[0] != JS_Undefined
+                        else list(obj)[:limit or None])
                 elif member == 'join':
                     assertion(isinstance(obj, list), 'must be applied on a list')
-                    assertion(len(argvals) == 1, 'takes exactly one argument')
-                    return argvals[0].join(obj)
+                    assertion(len(argvals) <= 1, 'takes at most one argument')
+                    return (',' if len(argvals) == 0 else argvals[0]).join(
+                        ('' if x in (None, JS_Undefined) else _js_toString(x))
+                        for x in obj)
                 elif member == 'reverse':
                     assertion(not argvals, 'does not take any arguments')
                     obj.reverse()

From 60835ca16c052eb00bb9bccd44f8843edac66a2e Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 12 Dec 2024 04:16:07 +0000
Subject: [PATCH 794/860] [jsinterp] Fix and improve "methods" * push, unshift
 return new length * impove edge cases for push/pop, shift/unshift, forEach,
 indexOf, charCodeAt * increase test coverage

---
 test/test_jsinterp.py  | 34 ++++++++++++++++++++++++++++++++++
 youtube_dl/jsinterp.py | 36 +++++++++++++++---------------------
 2 files changed, 49 insertions(+), 21 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index b6e87e9f1..07ed481d8 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -544,6 +544,40 @@ class TestJSInterpreter(unittest.TestCase):
         self._test('function f(){return "012345678".slice(-1, 1)}', '')
         self._test('function f(){return "012345678".slice(-3, -1)}', '67')
 
+    def test_pop(self):
+        # pop
+        self._test('function f(){var a = [0, 1, 2, 3, 4, 5, 6, 7, 8]; return [a.pop(), a]}',
+                   [8, [0, 1, 2, 3, 4, 5, 6, 7]])
+        self._test('function f(){return [].pop()}', JS_Undefined)
+        # push
+        self._test('function f(){var a = [0, 1, 2]; return [a.push(3, 4), a]}',
+                   [5, [0, 1, 2, 3, 4]])
+        self._test('function f(){var a = [0, 1, 2]; return [a.push(), a]}',
+                   [3, [0, 1, 2]])
+
+    def test_shift(self):
+        # shift
+        self._test('function f(){var a = [0, 1, 2, 3, 4, 5, 6, 7, 8]; return [a.shift(), a]}',
+                   [0, [1, 2, 3, 4, 5, 6, 7, 8]])
+        self._test('function f(){return [].shift()}', JS_Undefined)
+        # unshift
+        self._test('function f(){var a = [0, 1, 2]; return [a.unshift(3, 4), a]}',
+                   [5, [3, 4, 0, 1, 2]])
+        self._test('function f(){var a = [0, 1, 2]; return [a.unshift(), a]}',
+                   [3, [0, 1, 2]])
+
+    def test_forEach(self):
+        self._test('function f(){var ret = []; var l = [4, 2]; '
+                   'var log = function(e,i,a){ret.push([e,i,a]);}; '
+                   'l.forEach(log); '
+                   'return [ret.length, ret[0][0], ret[1][1], ret[0][2]]}',
+                   [2, 4, 1, [4, 2]])
+        self._test('function f(){var ret = []; var l = [4, 2]; '
+                   'var log = function(e,i,a){this.push([e,i,a]);}; '
+                   'l.forEach(log, ret); '
+                   'return [ret.length, ret[0][0], ret[1][1], ret[0][2]]}',
+                   [2, 4, 1, [4, 2]])
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index bec959946..0cfae4b28 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -1113,37 +1113,31 @@ class JSInterpreter(object):
                     index, how_many = map(int, (argvals + [len(obj)])[:2])
                     if index < 0:
                         index += len(obj)
-                    add_items = argvals[2:]
-                    res = []
-                    for _ in range(index, min(index + how_many, len(obj))):
-                        res.append(obj.pop(index))
-                    for i, item in enumerate(add_items):
-                        obj.insert(index + i, item)
+                    res = [obj.pop(index)
+                           for _ in range(index, min(index + how_many, len(obj)))]
+                    obj[index:index] = argvals[2:]
                     return res
-                elif member == 'unshift':
-                    assertion(isinstance(obj, list), 'must be applied on a list')
-                    assertion(argvals, 'takes one or more arguments')
-                    for item in reversed(argvals):
-                        obj.insert(0, item)
-                    return obj
-                elif member == 'pop':
+                elif member in ('shift', 'pop'):
                     assertion(isinstance(obj, list), 'must be applied on a list')
                     assertion(not argvals, 'does not take any arguments')
-                    if not obj:
-                        return
-                    return obj.pop()
+                    return obj.pop(0 if member == 'shift' else -1) if len(obj) > 0 else JS_Undefined
+                elif member == 'unshift':
+                    assertion(isinstance(obj, list), 'must be applied on a list')
+                    # not enforced: assertion(argvals, 'takes one or more arguments')
+                    obj[0:0] = argvals
+                    return len(obj)
                 elif member == 'push':
-                    assertion(argvals, 'takes one or more arguments')
+                    # not enforced: assertion(argvals, 'takes one or more arguments')
                     obj.extend(argvals)
-                    return obj
+                    return len(obj)
                 elif member == 'forEach':
                     assertion(argvals, 'takes one or more arguments')
-                    assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
+                    assertion(len(argvals) <= 2, 'takes at most 2 arguments')
                     f, this = (argvals + [''])[:2]
                     return [f((item, idx, obj), {'this': this}, allow_recursion) for idx, item in enumerate(obj)]
                 elif member == 'indexOf':
                     assertion(argvals, 'takes one or more arguments')
-                    assertion(len(argvals) <= 2, 'takes at-most 2 arguments')
+                    assertion(len(argvals) <= 2, 'takes at most 2 arguments')
                     idx, start = (argvals + [0])[:2]
                     try:
                         return obj.index(idx, start)
@@ -1152,7 +1146,7 @@ class JSInterpreter(object):
                 elif member == 'charCodeAt':
                     assertion(isinstance(obj, compat_str), 'must be applied on a string')
                     # assertion(len(argvals) == 1, 'takes exactly one argument') # but not enforced
-                    idx = argvals[0] if isinstance(argvals[0], int) else 0
+                    idx = argvals[0] if len(argvals) > 0 and isinstance(argvals[0], int) else 0
                     if idx >= len(obj):
                         return None
                     return ord(obj[idx])

From f487b4a02a74bb143a7b8f7cb26819b5b661a40b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 12 Dec 2024 04:21:53 +0000
Subject: [PATCH 795/860] [jsinterp] Strip /* comments */ when parsing * NB:
 _separate() is looking creaky

---
 test/test_jsinterp.py  | 10 +++++++++-
 youtube_dl/jsinterp.py | 20 ++++++++++++++++++--
 2 files changed, 27 insertions(+), 3 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 07ed481d8..12e7b9b94 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -160,7 +160,6 @@ class TestJSInterpreter(unittest.TestCase):
         self._test('function f(){var x = 20; x += 30 + 1; return x;}', 51)
         self._test('function f(){var x = 20; x -= 30 + 1; return x;}', -11)
 
-    @unittest.skip('Not yet fully implemented')
     def test_comments(self):
         self._test('''
             function f() {
@@ -179,6 +178,15 @@ class TestJSInterpreter(unittest.TestCase):
             }
         ''', 3)
 
+        self._test('''
+            function f() {
+                var x = ( /* 1 + */ 2 +
+                          /* 30 * 40 */
+                          50);
+                return x;
+            }
+        ''', 52)
+
     def test_precedence(self):
         self._test('''
             function f() {
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 0cfae4b28..ec8674936 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -488,9 +488,18 @@ class JSInterpreter(object):
         skipping = 0
         if skip_delims:
             skip_delims = variadic(skip_delims)
+        skip_txt = None
         for idx, char in enumerate(expr):
+            if skip_txt and idx <= skip_txt[1]:
+                continue
             paren_delta = 0
             if not in_quote:
+                if char == '/' and expr[idx:idx + 2] == '/*':
+                    # skip a comment
+                    skip_txt = expr[idx:].find('*/', 2)
+                    skip_txt = [idx, idx + skip_txt + 1] if skip_txt >= 2 else None
+                    if skip_txt:
+                        continue
                 if char in _MATCHING_PARENS:
                     counters[_MATCHING_PARENS[char]] += 1
                     paren_delta = 1
@@ -523,12 +532,19 @@ class JSInterpreter(object):
             if pos < delim_len:
                 pos += 1
                 continue
-            yield expr[start: idx - delim_len]
+            if skip_txt and skip_txt[0] >= start and skip_txt[1] <= idx - delim_len:
+                yield expr[start:skip_txt[0]] + expr[skip_txt[1] + 1: idx - delim_len]
+            else:
+                yield expr[start: idx - delim_len]
+            skip_txt = None
             start, pos = idx + 1, 0
             splits += 1
             if max_split and splits >= max_split:
                 break
-        yield expr[start:]
+        if skip_txt and skip_txt[0] >= start:
+            yield expr[start:skip_txt[0]] + expr[skip_txt[1] + 1:]
+        else:
+            yield expr[start:]
 
     @classmethod
     def _separate_at_paren(cls, expr, delim=None):

From ce1e556b8fde56eb84bec1580e9f07f45e721c4f Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 12 Dec 2024 04:29:13 +0000
Subject: [PATCH 796/860] [jsinterp] Add return hook for player `3bb1f723` *
 set var `_ytdl_do_not_return` to a specific value in the scope of a function
 * if an expression to be returned has that value, `return` becomes `void`

---
 youtube_dl/jsinterp.py | 33 +++++++++++++++++++++------------
 1 file changed, 21 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index ec8674936..7835187f5 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -368,7 +368,7 @@ class Debugger(object):
                 raise
             if cls.ENABLED and stmt.strip():
                 if should_ret or repr(ret) != stmt:
-                    cls.write(['->', '=>'][should_ret], repr(ret), '<-|', stmt, level=allow_recursion)
+                    cls.write(['->', '=>'][bool(should_ret)], repr(ret), '<-|', stmt, level=allow_recursion)
             return ret, should_ret
         return interpret_statement
 
@@ -603,7 +603,7 @@ class JSInterpreter(object):
 
     # used below
     _VAR_RET_THROW_RE = re.compile(r'''(?x)
-        (?P<var>(?:var|const|let)\s)|return(?:\s+|(?=["'])|$)|(?P<throw>throw\s+)
+        (?:(?P<var>var|const|let)\s+|(?P<ret>return)(?:\s+|(?=["'])|$)|(?P<throw>throw)\s+)
         ''')
     _COMPOUND_RE = re.compile(r'''(?x)
         (?P<try>try)\s*\{|
@@ -683,7 +683,7 @@ class JSInterpreter(object):
             expr = stmt[len(m.group(0)):].strip()
             if m.group('throw'):
                 raise JS_Throw(self.interpret_expression(expr, local_vars, allow_recursion))
-            should_return = not m.group('var')
+            should_return = 'return' if m.group('ret') else False
         if not expr:
             return None, should_return
 
@@ -968,14 +968,20 @@ class JSInterpreter(object):
             return _Infinity, should_return
 
         elif md.get('return'):
-            return local_vars[m.group('name')], should_return
+            ret = local_vars[m.group('name')]
+            # challenge may try to force returning the original value
+            # use an optional internal var to block this
+            if should_return == 'return':
+                if '_ytdl_do_not_return' not in local_vars:
+                    return ret, True
+                return (ret, True) if ret != local_vars['_ytdl_do_not_return'] else (ret, False)
+            else:
+                return ret, should_return
 
-        try:
+        with compat_contextlib_suppress(ValueError):
             ret = json.loads(js_to_json(expr))  # strict=True)
             if not md.get('attribute'):
                 return ret, should_return
-        except ValueError:
-            pass
 
         if md.get('indexing'):
             val = local_vars[m.group('in')]
@@ -1213,7 +1219,7 @@ class JSInterpreter(object):
             yield self.interpret_expression(v, local_vars, allow_recursion)
 
     def extract_object(self, objname):
-        _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')'''
+        _FUNC_NAME_RE = r'''(?:{n}|"{n}"|'{n}')'''.format(n=_NAME_RE)
         obj = {}
         fields = next(filter(None, (
             obj_m.group('fields') for obj_m in re.finditer(
@@ -1272,6 +1278,7 @@ class JSInterpreter(object):
 
     def extract_function_from_code(self, argnames, code, *global_stack):
         local_vars = {}
+
         while True:
             mobj = re.search(r'function\((?P<args>[^)]*)\)\s*{', code)
             if mobj is None:
@@ -1282,10 +1289,11 @@ class JSInterpreter(object):
                 [x.strip() for x in mobj.group('args').split(',')],
                 body, local_vars, *global_stack))
             code = code[:start] + name + remaining
+
         return self.build_function(argnames, code, local_vars, *global_stack)
 
-    def call_function(self, funcname, *args):
-        return self.extract_function(funcname)(args)
+    def call_function(self, funcname, *args, **kw_global_vars):
+        return self.extract_function(funcname)(args, kw_global_vars)
 
     @classmethod
     def build_arglist(cls, arg_text):
@@ -1304,8 +1312,9 @@ class JSInterpreter(object):
         global_stack = list(global_stack) or [{}]
         argnames = tuple(argnames)
 
-        def resf(args, kwargs={}, allow_recursion=100):
-            global_stack[0].update(zip_longest(argnames, args, fillvalue=None))
+        def resf(args, kwargs=None, allow_recursion=100):
+            kwargs = kwargs or {}
+            global_stack[0].update(zip_longest(argnames, args, fillvalue=JS_Undefined))
             global_stack[0].update(kwargs)
             var_stack = LocalNameSpace(*global_stack)
             ret, should_abort = self.interpret_statement(code.replace('\n', ' '), var_stack, allow_recursion - 1)

From b4469a0f652c450a81901795a8f522ae5457a1cf Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 7 Dec 2024 03:39:44 +0000
Subject: [PATCH 797/860] [YouTube] Handle player `3bb1f723` * fix signature
 code extraction * raise if n function returns input value * add new tests
 from yt-dlp

Co-authored-by: bashonly
---
 test/test_youtube_signature.py  | 37 +++++++++++++++++++++++++-------
 youtube_dl/extractor/youtube.py | 38 ++++++++++++++++++++++-----------
 2 files changed, 54 insertions(+), 21 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 56e92fac5..7d1ff90ba 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -12,6 +12,7 @@ import re
 import string
 
 from youtube_dl.compat import (
+    compat_contextlib_suppress,
     compat_open as open,
     compat_str,
     compat_urlretrieve,
@@ -50,23 +51,33 @@ _SIG_TESTS = [
     (
         'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflBb0OQx.js',
         84,
-        '123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQ0STUVWXYZ!"#$%&\'()*+,@./:;<=>'
+        '123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQ0STUVWXYZ!"#$%&\'()*+,@./:;<=>',
     ),
     (
         'https://s.ytimg.com/yts/jsbin/html5player-en_US-vfl9FYC6l.js',
         83,
-        '123456789abcdefghijklmnopqr0tuvwxyzABCDETGHIJKLMNOPQRS>UVWXYZ!"#$%&\'()*+,-./:;<=F'
+        '123456789abcdefghijklmnopqr0tuvwxyzABCDETGHIJKLMNOPQRS>UVWXYZ!"#$%&\'()*+,-./:;<=F',
     ),
     (
         'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflCGk6yw/html5player.js',
         '4646B5181C6C3020DF1D9C7FCFEA.AD80ABF70C39BD369CCCAE780AFBB98FA6B6CB42766249D9488C288',
-        '82C8849D94266724DC6B6AF89BBFA087EACCD963.B93C07FBA084ACAEFCF7C9D1FD0203C6C1815B6B'
+        '82C8849D94266724DC6B6AF89BBFA087EACCD963.B93C07FBA084ACAEFCF7C9D1FD0203C6C1815B6B',
     ),
     (
         'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js',
         '312AA52209E3623129A412D56A40F11CB0AF14AE.3EE09501CB14E3BCDC3B2AE808BF3F1D14E7FBF12',
         '112AA5220913623229A412D56A40F11CB0AF14AE.3EE0950FCB14EEBCDC3B2AE808BF331D14E7FBF3',
-    )
+    ),
+    (
+        'https://www.youtube.com/s/player/6ed0d907/player_ias.vflset/en_US/base.js',
+        '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+        'AOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL2QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0',
+    ),
+    (
+        'https://www.youtube.com/s/player/3bb1f723/player_ias.vflset/en_US/base.js',
+        '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+        'MyOSJXtKI3m-uME_jv7-pT12gOFC02RFkGoqWpzE0Cs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+    ),
 ]
 
 _NSIG_TESTS = [
@@ -142,6 +153,10 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/5a3b6271/player_ias.vflset/en_US/base.js',
         'B2j7f_UPT4rfje85Lu_e', 'm5DmNymaGQ5RdQ',
     ),
+    (
+        'https://www.youtube.com/s/player/7a062b77/player_ias.vflset/en_US/base.js',
+        'NRcE3y3mVtm_cV-W', 'VbsCYUATvqlt5w',
+    ),
     (
         'https://www.youtube.com/s/player/dac945fd/player_ias.vflset/en_US/base.js',
         'o8BkRxXhuYsBCWi6RplPdP', '3Lx32v_hmzTm6A',
@@ -154,6 +169,10 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/cfa9e7cb/player_ias.vflset/en_US/base.js',
         'qO0NiMtYQ7TeJnfFG2', 'k9cuJDHNS5O7kQ',
     ),
+    (
+        'https://www.youtube.com/s/player/8c7583ff/player_ias.vflset/en_US/base.js',
+        '1wWCVpRR96eAmMI87L', 'KSkWAVv1ZQxC3A',
+    ),
     (
         'https://www.youtube.com/s/player/b7910ca8/player_ias.vflset/en_US/base.js',
         '_hXMCwMt9qE310D', 'LoZMgkkofRMCZQ',
@@ -182,6 +201,10 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/b12cc44b/player_ias.vflset/en_US/base.js',
         'keLa5R2U00sR9SQK', 'N1OGyujjEwMnLw',
     ),
+    (
+        'https://www.youtube.com/s/player/3bb1f723/player_ias.vflset/en_US/base.js',
+        'gK15nzVyaXE9RsMP3z', 'ZFFWFLPWx9DEgQ',
+    ),
 ]
 
 
@@ -216,11 +239,9 @@ class TestSignature(unittest.TestCase):
             os.mkdir(self.TESTDATA_DIR)
 
     def tearDown(self):
-        try:
+        with compat_contextlib_suppress(OSError):
             for f in os.listdir(self.TESTDATA_DIR):
                 os.remove(f)
-        except OSError:
-            pass
 
 
 def t_factory(name, sig_func, url_pattern):
@@ -258,7 +279,7 @@ def n_sig(jscode, sig_input):
 
 
 make_sig_test = t_factory(
-    'signature', signature, re.compile(r'.*-(?P<id>[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$'))
+    'signature', signature, re.compile(r'.*(?:-|/player/)(?P<id>[a-zA-Z0-9_-]+)(?:/.+\.js|(?:/watch_as3|/html5player)?\.[a-z]+)$'))
 for test_spec in _SIG_TESTS:
     make_sig_test(*test_spec)
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6fe520e9a..d633032ae 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1579,19 +1579,26 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         self.to_screen('Extracted signature function:\n' + code)
 
     def _parse_sig_js(self, jscode):
+        # Examples where `sig` is funcname:
+        # sig=function(a){a=a.split(""); ... ;return a.join("")};
+        # ;c&&(c=sig(decodeURIComponent(c)),a.set(b,encodeURIComponent(c)));return a};
+        # {var l=f,m=h.sp,n=sig(decodeURIComponent(h.s));l.set(m,encodeURIComponent(n))}
+        # sig=function(J){J=J.split(""); ... ;return J.join("")};
+        # ;N&&(N=sig(decodeURIComponent(N)),J.set(R,encodeURIComponent(N)));return J};
+        # {var H=u,k=f.sp,v=sig(decodeURIComponent(f.s));H.set(k,encodeURIComponent(v))}
         funcname = self._search_regex(
-            (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
+            (r'\b(?P<var>[a-zA-Z0-9$]+)&&\((?P=var)=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\((?P=var)\)\)',
+             r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*(?P<arg>[a-zA-Z0-9$]+)\s*\)\s*{\s*(?P=arg)\s*=\s*(?P=arg)\.split\(\s*""\s*\)\s*;\s*[^}]+;\s*return\s+(?P=arg)\.join\(\s*""\s*\)',
+             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\))?',
+             # Old patterns
+             r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
-             r'\bc&&\(c=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)',
-             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\))?',
-             r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)',
              # Obsolete patterns
              r'("|\')signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
              r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
              r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
             jscode, 'Initial JS player signature function name', group='sig')
 
@@ -1658,6 +1665,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     def _extract_n_function_name(self, jscode):
         func_name, idx = self._search_regex(
+            # (R="nn"[+J.Z],mW(J),N=J.K[R]||null)&&(N=narray[idx](N),J.set(R,N))}};
             # new: (b=String.fromCharCode(110),c=a.get(b))&&c=nfunc[idx](c)
             # or:  (b="nn"[+a.D],c=a.get(b))&&(c=nfunc[idx](c)
             # or:  (PL(a),b=a.j.n||null)&&(b=nfunc[idx](b)
@@ -1666,7 +1674,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             # older: (b=a.get("n"))&&(b=nfunc(b)
             r'''(?x)
                 \((?:[\w$()\s]+,)*?\s*      # (
-                (?P<b>[a-z])\s*=\s*         # b=
+                (?P<b>[a-zA-Z])\s*=\s*      # b=, R=
                 (?:
                     (?:                     # expect ,c=a.get(b) (etc)
                         String\s*\.\s*fromCharCode\s*\(\s*110\s*\)|
@@ -1679,7 +1687,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                        (?:\.\s*[\w$]+\s*|\[\s*[\w$]+\s*]\s*)*?
                        (?:\.\s*n|\[\s*"n"\s*]|\.\s*get\s*\(\s*"n"\s*\))
                        |                    # ,c=a.get(b)
-                       ,\s*(?P<c>[a-z])\s*=\s*[a-z]\s*
+                       ,\s*(?P<c>[a-zA-Z])\s*=\s*[a-zA-Z]\s*
                        (?:\.\s*[\w$]+\s*|\[\s*[\w$]+\s*]\s*)*?
                        (?:\[\s*(?P=b)\s*]|\.\s*get\s*\(\s*(?P=b)\s*\))
                    )
@@ -1697,15 +1705,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 r'''(?xs)
                     (?:(?<=[^\w$])|^)       # instead of \b, which ignores $
                     (?P<name>(?!\d)[a-zA-Z\d_$]+)\s*=\s*function\((?!\d)[a-zA-Z\d_$]+\)
-                    \s*\{(?:(?!};).)+?["']enhanced_except_
+                    \s*\{(?:(?!};).)+?(?:
+                        ["']enhanced_except_ |
+                        return\s*(?P<q>"|')[a-zA-Z\d-]+_w8_(?P=q)\s*\+\s*[a-zA-Z0-9_$]+
+                    )
                 ''', jscode, 'Initial JS player n function name', group='name')
         if not idx:
             return func_name
 
-        return self._parse_json(self._search_regex(
-            r'var\s+{0}\s*=\s*(\[.+?\])\s*[,;]'.format(re.escape(func_name)), jscode,
-            'Initial JS player n function list ({0}.{1})'.format(func_name, idx)),
-            func_name, transform_source=js_to_json)[int(idx)]
+        return self._search_json(
+            r'var\s+{0}\s*='.format(re.escape(func_name)), jscode,
+            'Initial JS player n function list ({0}.{1})'.format(func_name, idx),
+            func_name, contains_pattern=r'\[[\s\S]+\]', end_pattern='[,;]',
+            transform_source=js_to_json)[int(idx)]
 
     def _extract_n_function_code(self, video_id, player_url):
         player_id = self._extract_player_info(player_url)
@@ -1734,7 +1746,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             except Exception as e:
                 raise JSInterpreter.Exception(traceback.format_exc(), cause=e)
 
-            if ret.startswith('enhanced_except_'):
+            if ret.startswith('enhanced_except_') or ret.endswith(s):
                 raise JSInterpreter.Exception('Signature function returned an exception')
             return ret
 

From eed784e15f6066b152a3cce8db6fe3f059290b22 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 12 Dec 2024 04:38:23 +0000
Subject: [PATCH 798/860] [YouTube] Pass nsig value as return hook, fixes
 player `3bb1f723`

---
 test/test_youtube_signature.py  | 4 +++-
 youtube_dl/extractor/youtube.py | 2 +-
 2 files changed, 4 insertions(+), 2 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 7d1ff90ba..1c2ba848e 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -1,4 +1,5 @@
 #!/usr/bin/env python
+# coding: utf-8
 
 from __future__ import unicode_literals
 
@@ -275,7 +276,8 @@ def signature(jscode, sig_input):
 
 def n_sig(jscode, sig_input):
     funcname = YoutubeIE(FakeYDL())._extract_n_function_name(jscode)
-    return JSInterpreter(jscode).call_function(funcname, sig_input)
+    return JSInterpreter(jscode).call_function(
+        funcname, sig_input, _ytdl_do_not_return=sig_input)
 
 
 make_sig_test = t_factory(
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index d633032ae..55881f4d1 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1740,7 +1740,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         def extract_nsig(s):
             try:
-                ret = func([s])
+                ret = func([s], kwargs={'_ytdl_do_not_return': s})
             except JSInterpreter.Exception:
                 raise
             except Exception as e:

From cd7c7b5edbf020730f106aabaa15ca1a7de22e44 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 13 Dec 2024 03:09:29 +0000
Subject: [PATCH 799/860] [YouTube] Simplify pattern for nsig function name
 extraction

---
 test/test_youtube_signature.py  |  4 +++
 youtube_dl/extractor/youtube.py | 46 ++++++++++++++-------------------
 2 files changed, 23 insertions(+), 27 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 1c2ba848e..1d6bce67c 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -206,6 +206,10 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/3bb1f723/player_ias.vflset/en_US/base.js',
         'gK15nzVyaXE9RsMP3z', 'ZFFWFLPWx9DEgQ',
     ),
+    (
+        'https://www.youtube.com/s/player/f8f53e1a/player_ias.vflset/en_US/base.js',
+        'VTQOUOv0mCIeJ7i8kZB', 'kcfD8wy0sNLyNQ',
+    ),
 ]
 
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 55881f4d1..f59d1e36a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1665,37 +1665,29 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     def _extract_n_function_name(self, jscode):
         func_name, idx = self._search_regex(
+            # (y=NuD(),Mw(k),q=k.Z[y]||null)&&(q=narray[idx](q),k.set(y,q),k.V||NuD(''))}};
             # (R="nn"[+J.Z],mW(J),N=J.K[R]||null)&&(N=narray[idx](N),J.set(R,N))}};
-            # new: (b=String.fromCharCode(110),c=a.get(b))&&c=nfunc[idx](c)
-            # or:  (b="nn"[+a.D],c=a.get(b))&&(c=nfunc[idx](c)
-            # or:  (PL(a),b=a.j.n||null)&&(b=nfunc[idx](b)
+            # or:  (b=String.fromCharCode(110),c=a.get(b))&&c=narray[idx](c)
+            # or:  (b="nn"[+a.D],c=a.get(b))&&(c=narray[idx](c)
+            # or:  (PL(a),b=a.j.n||null)&&(b=narray[idx](b)
             # or:  (b="nn"[+a.D],vL(a),c=a.j[b]||null)&&(c=narray[idx](c),a.set(b,c),narray.length||nfunc("")
-            # old: (b=a.get("n"))&&(b=nfunc[idx](b)(?P<c>[a-z])\s*=\s*[a-z]\s*
+            # old: (b=a.get("n"))&&(b=narray[idx](b)(?P<c>[a-z])\s*=\s*[a-z]\s*
             # older: (b=a.get("n"))&&(b=nfunc(b)
             r'''(?x)
-                \((?:[\w$()\s]+,)*?\s*      # (
-                (?P<b>[a-zA-Z])\s*=\s*      # b=, R=
-                (?:
-                    (?:                     # expect ,c=a.get(b) (etc)
-                        String\s*\.\s*fromCharCode\s*\(\s*110\s*\)|
-                        "n+"\[\s*\+?s*[\w$.]+\s*]
-                    )\s*(?:,[\w$()\s]+(?=,))*|
-                       (?P<old>[\w$]+)      # a (old[er])
-                   )\s*
-                   (?(old)
-                                            # b.get("n")
-                       (?:\.\s*[\w$]+\s*|\[\s*[\w$]+\s*]\s*)*?
-                       (?:\.\s*n|\[\s*"n"\s*]|\.\s*get\s*\(\s*"n"\s*\))
-                       |                    # ,c=a.get(b)
-                       ,\s*(?P<c>[a-zA-Z])\s*=\s*[a-zA-Z]\s*
-                       (?:\.\s*[\w$]+\s*|\[\s*[\w$]+\s*]\s*)*?
-                       (?:\[\s*(?P=b)\s*]|\.\s*get\s*\(\s*(?P=b)\s*\))
-                   )
-                                            # interstitial junk
-                   \s*(?:\|\|\s*null\s*)?(?:\)\s*)?&&\s*(?:\(\s*)?
-               (?(c)(?P=c)|(?P=b))\s*=\s*   # [c|b]=
-                                            # nfunc|nfunc[idx]
-                   (?P<nfunc>[a-zA-Z_$][\w$]*)(?:\s*\[(?P<idx>\d+)\])?\s*\(\s*[\w$]+\s*\)
+                # (expr, ...,
+                \((?:(?:\s*[\w$]+\s*=)?(?:[\w$"+\.\s(\[]+(?:[)\]]\s*)?),)*
+                  # b=...
+                  (?P<b>[\w$]+)\s*=\s*(?!(?P=b)[^\w$])[\w$]+\s*(?:(?:
+                    \.\s*[\w$]+ |
+                    \[\s*[\w$]+\s*\] |
+                    \.\s*get\s*\(\s*[\w$"]+\s*\)
+                  )\s*){,2}(?:\s*\|\|\s*null(?=\s*\)))?\s*
+                \)\s*&&\s*\(        # ...)&&(
+                # b = nfunc, b = narray[idx]
+                (?P=b)\s*=\s*(?P<nfunc>[\w$]+)\s*
+                    (?:\[\s*(?P<idx>[\w$]+)\s*\]\s*)?
+                    # (...)
+                    \(\s*[\w$]+\s*\)
             ''', jscode, 'Initial JS player n function name', group=('nfunc', 'idx'),
             default=(None, None))
         # thx bashonly: yt-dlp/yt-dlp/pull/10611

From eeafbbc3e59bcd6513053f5c942a55e2ff309488 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 13 Dec 2024 12:16:31 +0000
Subject: [PATCH 800/860] [YouTube] Fix signature function extraction for
 `2f1832d2` * `_` was omitted from patterns * thx yt-dlp/yt-dlp#11801

Co-authored-by: bashonly
---
 test/test_youtube_signature.py  |  9 +++++++++
 youtube_dl/extractor/youtube.py | 24 ++++++++++++------------
 2 files changed, 21 insertions(+), 12 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 1d6bce67c..fcbc9d7a8 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -79,6 +79,11 @@ _SIG_TESTS = [
         '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
         'MyOSJXtKI3m-uME_jv7-pT12gOFC02RFkGoqWpzE0Cs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
     ),
+    (
+        'https://www.youtube.com/s/player/2f1832d2/player_ias.vflset/en_US/base.js',
+        '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+        '0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xxAj7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJ2OySqa0q',
+    ),
 ]
 
 _NSIG_TESTS = [
@@ -210,6 +215,10 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/f8f53e1a/player_ias.vflset/en_US/base.js',
         'VTQOUOv0mCIeJ7i8kZB', 'kcfD8wy0sNLyNQ',
     ),
+    (
+        'https://www.youtube.com/s/player/2f1832d2/player_ias.vflset/en_US/base.js',
+        'YWt1qdbe8SAfkoPHW5d', 'RrRjWQOJmBiP',
+    ),
 ]
 
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f59d1e36a..7e0ba04c0 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1587,19 +1587,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         # ;N&&(N=sig(decodeURIComponent(N)),J.set(R,encodeURIComponent(N)));return J};
         # {var H=u,k=f.sp,v=sig(decodeURIComponent(f.s));H.set(k,encodeURIComponent(v))}
         funcname = self._search_regex(
-            (r'\b(?P<var>[a-zA-Z0-9$]+)&&\((?P=var)=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\((?P=var)\)\)',
-             r'(?P<sig>[a-zA-Z0-9$]+)\s*=\s*function\(\s*(?P<arg>[a-zA-Z0-9$]+)\s*\)\s*{\s*(?P=arg)\s*=\s*(?P=arg)\.split\(\s*""\s*\)\s*;\s*[^}]+;\s*return\s+(?P=arg)\.join\(\s*""\s*\)',
-             r'(?:\b|[^a-zA-Z0-9$])(?P<sig>[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\))?',
+            (r'\b(?P<var>[\w$]+)&&\((?P=var)=(?P<sig>[\w$]{2,})\(decodeURIComponent\((?P=var)\)\)',
+             r'(?P<sig>[\w$]+)\s*=\s*function\(\s*(?P<arg>[\w$]+)\s*\)\s*{\s*(?P=arg)\s*=\s*(?P=arg)\.split\(\s*""\s*\)\s*;\s*[^}]+;\s*return\s+(?P=arg)\.join\(\s*""\s*\)',
+             r'(?:\b|[^\w$])(?P<sig>[\w$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[\w$]{2}\.[\w$]{2}\(a,\d+\))?',
              # Old patterns
-             r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'\bm=(?P<sig>[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)',
+             r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[\w$]+)\(',
+             r'\b[\w]+\s*&&\s*[\w]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P<sig>[\w$]+)\(',
+             r'\bm=(?P<sig>[\w$]{2,})\(decodeURIComponent\(h\.s\)\)',
              # Obsolete patterns
-             r'("|\')signature\1\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'\.sig\|\|(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[a-zA-Z0-9$]+)\(',
-             r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[a-zA-Z0-9$]+)\('),
+             r'("|\')signature\1\s*,\s*(?P<sig>[\w$]+)\(',
+             r'\.sig\|\|(?P<sig>[\w$]+)\(',
+             r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P<sig>[\w$]+)\(',
+             r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P<sig>[\w$]+)\(',
+             r'\bc\s*&&\s*[\w]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P<sig>[\w$]+)\('),
             jscode, 'Initial JS player signature function name', group='sig')
 
         jsi = JSInterpreter(jscode)
@@ -1699,7 +1699,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     (?P<name>(?!\d)[a-zA-Z\d_$]+)\s*=\s*function\((?!\d)[a-zA-Z\d_$]+\)
                     \s*\{(?:(?!};).)+?(?:
                         ["']enhanced_except_ |
-                        return\s*(?P<q>"|')[a-zA-Z\d-]+_w8_(?P=q)\s*\+\s*[a-zA-Z0-9_$]+
+                        return\s*(?P<q>"|')[a-zA-Z\d-]+_w8_(?P=q)\s*\+\s*[\w$]+
                     )
                 ''', jscode, 'Initial JS player n function name', group='name')
         if not idx:

From d55d1f423d6473ae6a9e13462d94bad1d71d28e0 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 14 Dec 2024 11:18:34 +0000
Subject: [PATCH 801/860] [YouTube] Always extract using MWEB API client *
 temporary fix-up for 403 on download * MWEB parameters from yt-dlp 2024-12-06

---
 youtube_dl/extractor/youtube.py | 84 ++++++++++++++++++++++++++++++---
 1 file changed, 77 insertions(+), 7 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 7e0ba04c0..1f83acf7c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -3,11 +3,13 @@
 from __future__ import unicode_literals
 
 import collections
+import hashlib
 import itertools
 import json
 import os.path
 import random
 import re
+import time
 import traceback
 
 from .common import InfoExtractor, SearchInfoExtractor
@@ -290,6 +292,33 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
     _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|</script|\n)'
 
+    _SAPISID = None
+
+    def _generate_sapisidhash_header(self, origin='https://www.youtube.com'):
+        time_now = round(time.time())
+        if self._SAPISID is None:
+            yt_cookies = self._get_cookies('https://www.youtube.com')
+            # Sometimes SAPISID cookie isn't present but __Secure-3PAPISID is.
+            # See: https://github.com/yt-dlp/yt-dlp/issues/393
+            sapisid_cookie = dict_get(
+                yt_cookies, ('__Secure-3PAPISID', 'SAPISID'))
+            if sapisid_cookie and sapisid_cookie.value:
+                self._SAPISID = sapisid_cookie.value
+                self.write_debug('Extracted SAPISID cookie')
+                # SAPISID cookie is required if not already present
+                if not yt_cookies.get('SAPISID'):
+                    self.write_debug('Copying __Secure-3PAPISID cookie to SAPISID cookie')
+                    self._set_cookie(
+                        '.youtube.com', 'SAPISID', self._SAPISID, secure=True, expire_time=time_now + 3600)
+            else:
+                self._SAPISID = False
+        if not self._SAPISID:
+            return None
+        # SAPISIDHASH algorithm from https://stackoverflow.com/a/32065323
+        sapisidhash = hashlib.sha1(
+            '{0} {1} {2}'.format(time_now, self._SAPISID, origin).encode('utf-8')).hexdigest()
+        return 'SAPISIDHASH {0}_{1}'.format(time_now, sapisidhash)
+
     def _call_api(self, ep, query, video_id, fatal=True, headers=None):
         data = self._DEFAULT_API_DATA.copy()
         data.update(query)
@@ -1914,9 +1943,50 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             player_response = self._extract_yt_initial_variable(
                 webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE,
                 video_id, 'initial player response')
-        if not player_response:
+        if False and not player_response:
             player_response = self._call_api(
                 'player', {'videoId': video_id}, video_id)
+        if True or not player_response:
+            origin = 'https://www.youtube.com'
+            pb_context = {'html5Preference': 'HTML5_PREF_WANTS'}
+
+            player_url = self._extract_player_url(webpage)
+            ytcfg = self._extract_ytcfg(video_id, webpage)
+            sts = self._extract_signature_timestamp(video_id, player_url, ytcfg)
+            if sts:
+                pb_context['signatureTimestamp'] = sts
+
+            query = {
+                'playbackContext': {
+                    'contentPlaybackContext': pb_context,
+                    'contentCheckOk': True,
+                    'racyCheckOk': True,
+                },
+                'context': {
+                    'client': {
+                        'clientName': 'MWEB',
+                        'clientVersion': '2.20241202.07.00',
+                        'hl': 'en',
+                        'userAgent': 'Mozilla/5.0 (iPad; CPU OS 16_7_10 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1,gzip(gfe)',
+                        'timeZone': 'UTC',
+                        'utcOffsetMinutes': 0,
+                    },
+                },
+                'videoId': video_id,
+            }
+            headers = {
+                'X-YouTube-Client-Name': '2',
+                'X-YouTube-Client-Version': '2.20241202.07.00',
+                'Origin': origin,
+                'Sec-Fetch-Mode': 'navigate',
+                'User-Agent': query['context']['client']['userAgent'],
+            }
+            auth = self._generate_sapisidhash_header(origin)
+            if auth is not None:
+                headers['Authorization'] = auth
+                headers['X-Origin'] = origin
+
+            player_response = self._call_api('player', query, video_id, fatal=False, headers=headers)
 
         def is_agegated(playability):
             if not isinstance(playability, dict):
@@ -2223,12 +2293,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         formats.append(f)
 
         playable_formats = [f for f in formats if not f.get('has_drm')]
-        if formats and not playable_formats:
-            # If there are no formats that definitely don't have DRM, all have DRM
-            self.report_drm(video_id)
-        formats[:] = playable_formats
-
-        if not formats:
+        if formats:
+            if not playable_formats:
+                # If there are no formats that definitely don't have DRM, all have DRM
+                self.report_drm(video_id)
+            formats[:] = playable_formats
+        else:
             if streaming_data.get('licenseInfos'):
                 raise ExtractorError(
                     'This video is DRM protected.', expected=True)

From 176fc2cb003b7a74f5781a64a4e1ce32e740c149 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 31 Dec 2024 14:51:29 +0000
Subject: [PATCH 802/860] [YouTube] Avoid early crash if webpage can't be read
 * see issue #33013

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1f83acf7c..7ea30fd40 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1951,7 +1951,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             pb_context = {'html5Preference': 'HTML5_PREF_WANTS'}
 
             player_url = self._extract_player_url(webpage)
-            ytcfg = self._extract_ytcfg(video_id, webpage)
+            ytcfg = self._extract_ytcfg(video_id, webpage or '')
             sts = self._extract_signature_timestamp(video_id, player_url, ytcfg)
             if sts:
                 pb_context['signatureTimestamp'] = sts

From ab7c61ca29ed1d1216d463d01794eb112a9144d5 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 6 Jan 2025 01:22:16 +0000
Subject: [PATCH 803/860] [YouTube] Apply code style changes, trailing commas,
 etc

---
 youtube_dl/extractor/youtube.py | 60 +++++++++++++++++----------------
 1 file changed, 31 insertions(+), 29 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 7ea30fd40..e9603d155 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -9,6 +9,7 @@ import json
 import os.path
 import random
 import re
+import string
 import time
 import traceback
 
@@ -67,6 +68,7 @@ from ..utils import (
 
 class YoutubeBaseInfoExtractor(InfoExtractor):
     """Provide base functions for Youtube extractors"""
+
     _LOGIN_URL = 'https://accounts.google.com/ServiceLogin'
     _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge'
 
@@ -138,7 +140,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                 [2, 1, None, 1,
                  'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn',
                  None, [], 4],
-                1, [None, None, []], None, None, None, True
+                1, [None, None, []], None, None, None, True,
             ],
             username,
         ]
@@ -160,7 +162,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             None, 1, None, [1, None, None, None, [password, None, True]],
             [
                 None, None, [2, 1, None, 1, 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn', None, [], 4],
-                1, [None, None, []], None, None, None, True
+                1, [None, None, []], None, None, None, True,
             ]]
 
         challenge_results = req(
@@ -213,7 +215,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                     user_hash, None, 2, None,
                     [
                         9, None, None, None, None, None, None, None,
-                        [None, tfa_code, True, 2]
+                        [None, tfa_code, True, 2],
                     ]]
 
                 tfa_results = req(
@@ -284,7 +286,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'client': {
                 'clientName': 'WEB',
                 'clientVersion': '2.20201021.03.00',
-            }
+            },
         },
     }
 
@@ -385,7 +387,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                 'client': {
                     'clientName': 'WEB',
                     'clientVersion': '2.20201021.03.00',
-                }
+                },
             },
             'query': query,
         }
@@ -462,7 +464,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             #       (HTML, videodetails, metadata, renderers)
             'name': ('content', 'author', (('ownerChannelName', None), 'title'), ['text']),
             'url': ('href', 'ownerProfileUrl', 'vanityChannelUrl',
-                    ['navigationEndpoint', 'browseEndpoint', 'canonicalBaseUrl'])
+                    ['navigationEndpoint', 'browseEndpoint', 'canonicalBaseUrl']),
         }
         if any((videodetails, metadata, renderers)):
             result = (
@@ -671,7 +673,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO',
                 'description': '',
                 'uploader': '8KVIDEO',
-                'title': 'UHDTV TEST 8K VIDEO.mp4'
+                'title': 'UHDTV TEST 8K VIDEO.mp4',
             },
             'params': {
                 'youtube_include_dash_manifest': True,
@@ -711,7 +713,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@theamazingatheist',
                 'title': 'Burning Everyone\'s Koran',
                 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms \r\n\r\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html',
-            }
+            },
         },
         # Age-gated videos
         {
@@ -839,7 +841,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             },
             'expected_warnings': [
                 'DASH manifest missing',
-            ]
+            ],
         },
         # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431)
         {
@@ -1820,8 +1822,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         # cpn generation algorithm is reverse engineered from base.js.
         # In fact it works even with dummy cpn.
-        CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_'
-        cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16))
+        CPN_ALPHABET = string.ascii_letters + string.digits + '-_'
+        cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(16))
 
         # more consistent results setting it to right before the end
         qs = parse_qs(playback_url)
@@ -1881,8 +1883,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         mobj = re.match(cls._VALID_URL, url, re.VERBOSE)
         if mobj is None:
             raise ExtractorError('Invalid URL: %s' % url)
-        video_id = mobj.group(2)
-        return video_id
+        return mobj.group(2)
 
     def _extract_chapters_from_json(self, data, video_id, duration):
         chapters_list = try_get(
@@ -2035,7 +2036,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             headers = {
                 'X-YouTube-Client-Name': '85',
                 'X-YouTube-Client-Version': '2.0',
-                'Origin': 'https://www.youtube.com'
+                'Origin': 'https://www.youtube.com',
             }
 
             video_info = self._call_api('player', query, video_id, fatal=False, headers=headers)
@@ -2064,8 +2065,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             return ''.join([r['text'] for r in runs if isinstance(r.get('text'), compat_str)])
 
         search_meta = (
-            lambda x: self._html_search_meta(x, webpage, default=None)) \
-            if webpage else lambda x: None
+            (lambda x: self._html_search_meta(x, webpage, default=None))
+            if webpage else lambda _: None)
 
         video_details = player_response.get('videoDetails') or {}
         microformat = try_get(
@@ -2137,7 +2138,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         def build_fragments(f):
             return LazyList({
                 'url': update_url_query(f['url'], {
-                    'range': '{0}-{1}'.format(range_start, min(range_start + CHUNK_SIZE - 1, f['filesize']))
+                    'range': '{0}-{1}'.format(range_start, min(range_start + CHUNK_SIZE - 1, f['filesize'])),
                 })
             } for range_start in range(0, f['filesize'], CHUNK_SIZE))
 
@@ -2236,7 +2237,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'protocol': 'http_dash_segments',
                     'fragments': build_fragments(dct),
                 } if dct['filesize'] else {
-                    'downloader_options': {'http_chunk_size': CHUNK_SIZE}  # No longer useful?
+                    'downloader_options': {'http_chunk_size': CHUNK_SIZE},  # No longer useful?
                 })
 
             formats.append(dct)
@@ -2454,7 +2455,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             info['subtitles'] = subtitles
 
         parsed_url = compat_urllib_parse_urlparse(url)
-        for component in [parsed_url.fragment, parsed_url.query]:
+        for component in (parsed_url.fragment, parsed_url.query):
             query = compat_parse_qs(component)
             for k, v in query.items():
                 for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]:
@@ -2684,7 +2685,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'title': 'Super Cooper Shorts - Shorts',
             'uploader': 'Super Cooper Shorts',
             'uploader_id': '@SuperCooperShorts',
-        }
+        },
     }, {
         # Channel that does not have a Shorts tab. Test should just download videos on Home tab instead
         'url': 'https://www.youtube.com/@emergencyawesome/shorts',
@@ -2738,7 +2739,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'description': 'md5:609399d937ea957b0f53cbffb747a14c',
             'uploader': 'ThirstForScience',
             'uploader_id': '@ThirstForScience',
-        }
+        },
     }, {
         'url': 'https://www.youtube.com/c/ChristophLaimer/playlists',
         'only_matching': True,
@@ -3037,7 +3038,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'uploader': '3Blue1Brown',
             'uploader_id': '@3blue1brown',
             'channel_id': 'UCYO_jab_esuFRV4b17AJtAw',
-        }
+        },
     }]
 
     @classmethod
@@ -3335,7 +3336,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             'client': {
                 'clientName': 'WEB',
                 'clientVersion': client_version,
-            }
+            },
         }
         visitor_data = try_get(context, lambda x: x['client']['visitorData'], compat_str)
 
@@ -3354,7 +3355,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                 headers['x-goog-visitor-id'] = visitor_data
             data['continuation'] = continuation['continuation']
             data['clickTracking'] = {
-                'clickTrackingParams': continuation['itct']
+                'clickTrackingParams': continuation['itct'],
             }
             count = 0
             retries = 3
@@ -3613,7 +3614,7 @@ class YoutubePlaylistIE(InfoExtractor):
             'uploader': 'milan',
             'uploader_id': '@milan5503',
             'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw',
-        }
+        },
     }, {
         'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl',
         'playlist_mincount': 455,
@@ -3623,7 +3624,7 @@ class YoutubePlaylistIE(InfoExtractor):
             'uploader': 'LBK',
             'uploader_id': '@music_king',
             'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA',
-        }
+        },
     }, {
         'url': 'TLGGrESM50VT6acwMjAyMjAxNw',
         'only_matching': True,
@@ -3734,7 +3735,7 @@ class YoutubeSearchIE(SearchInfoExtractor, YoutubeBaseInfoExtractor):
         'info_dict': {
             'id': 'youtube-dl test video',
             'title': 'youtube-dl test video',
-        }
+        },
     }]
 
     def _get_n_results(self, query, n):
@@ -3754,7 +3755,7 @@ class YoutubeSearchDateIE(YoutubeSearchIE):
         'info_dict': {
             'id': 'youtube-dl test video',
             'title': 'youtube-dl test video',
-        }
+        },
     }]
 
 
@@ -3769,7 +3770,7 @@ class YoutubeSearchURLIE(YoutubeBaseInfoExtractor):
             'id': 'youtube-dl test video',
             'title': 'youtube-dl test video',
         },
-        'params': {'playlistend': 5}
+        'params': {'playlistend': 5},
     }, {
         'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB',
         'only_matching': True,
@@ -3785,6 +3786,7 @@ class YoutubeSearchURLIE(YoutubeBaseInfoExtractor):
 class YoutubeFeedsInfoExtractor(YoutubeTabIE):
     """
     Base class for feed extractors
+
     Subclasses must define the _FEED_NAME property.
     """
     _LOGIN_REQUIRED = True

From 00ad2b8ca12d4f9b830ed83876d0d1ab3d698675 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 6 Jan 2025 01:24:30 +0000
Subject: [PATCH 804/860] [YouTube] Refactor subtitle processing * move to
 internal function * use `traverse-obj()`

---
 youtube_dl/extractor/youtube.py | 46 +++++++++++++++++----------------
 1 file changed, 24 insertions(+), 22 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e9603d155..56957a661 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2415,9 +2415,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             'is_live': is_live,
         }
 
-        pctr = try_get(
+        pctr = traverse_obj(
             player_response,
-            lambda x: x['captions']['playerCaptionsTracklistRenderer'], dict)
+            ('captions', 'playerCaptionsTracklistRenderer', T(dict)))
         if pctr:
             def process_language(container, base_url, lang_code, query):
                 lang_subs = []
@@ -2431,28 +2431,30 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     })
                 container[lang_code] = lang_subs
 
-            subtitles = {}
-            for caption_track in (pctr.get('captionTracks') or []):
-                base_url = caption_track.get('baseUrl')
-                if not base_url:
-                    continue
-                if caption_track.get('kind') != 'asr':
-                    lang_code = caption_track.get('languageCode')
-                    if not lang_code:
+            def process_subtitles():
+                subtitles = {}
+                for caption_track in traverse_obj(pctr, (
+                        'captionTracks', lambda _, v: v.get('baseUrl'))):
+                    if not base_url:
                         continue
-                    process_language(
-                        subtitles, base_url, lang_code, {})
-                    continue
-                automatic_captions = {}
-                for translation_language in (pctr.get('translationLanguages') or []):
-                    translation_language_code = translation_language.get('languageCode')
-                    if not translation_language_code:
+                    if caption_track.get('kind') != 'asr':
+                        lang_code = caption_track.get('languageCode')
+                        if not lang_code:
+                            continue
+                        process_language(
+                            subtitles, base_url, lang_code, {})
                         continue
-                    process_language(
-                        automatic_captions, base_url, translation_language_code,
-                        {'tlang': translation_language_code})
-                info['automatic_captions'] = automatic_captions
-            info['subtitles'] = subtitles
+                    automatic_captions = {}
+                    for translation_language in traverse_obj(pctr, (
+                            'translationLanguages', lambda _, v: v.get('languageCode'))):
+                        translation_language_code = translation_language['languageCode']
+                        process_language(
+                            automatic_captions, base_url, translation_language_code,
+                            {'tlang': translation_language_code})
+                    info['automatic_captions'] = automatic_captions
+                info['subtitles'] = subtitles
+
+            process_subtitles()
 
         parsed_url = compat_urllib_parse_urlparse(url)
         for component in (parsed_url.fragment, parsed_url.query):

From 1036478d130c5f2001eca2d7d12558abe601d933 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 6 Jan 2025 01:39:04 +0000
Subject: [PATCH 805/860] [YouTube] Endure subtitle URLs are complete * WEB
 URLs are, MWEB not * resolves #33017

---
 youtube_dl/extractor/youtube.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 56957a661..6171df84a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2435,6 +2435,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 subtitles = {}
                 for caption_track in traverse_obj(pctr, (
                         'captionTracks', lambda _, v: v.get('baseUrl'))):
+                    base_url = self._yt_urljoin(caption_track['baseUrl'])
                     if not base_url:
                         continue
                     if caption_track.get('kind') != 'asr':

From 21fff051217d1c14a7d50fa752052eadbcafee4e Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 15 Jan 2025 03:19:15 +0000
Subject: [PATCH 806/860] [YouTube] Switch to TV API client * thx
 yt-dlp/yt-dlp#12059

---
 youtube_dl/extractor/youtube.py | 102 ++++++++++++++++++++++++++------
 1 file changed, 83 insertions(+), 19 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6171df84a..1424277ac 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -31,7 +31,9 @@ from ..utils import (
     dict_get,
     error_to_compat_str,
     ExtractorError,
+    filter_dict,
     float_or_none,
+    get_first,
     extract_attributes,
     get_element_by_attribute,
     int_or_none,
@@ -82,6 +84,34 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
 
     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM)'
 
+    _INNERTUBE_CLIENTS = {
+        # mweb has 'ultralow' formats
+        # See: https://github.com/yt-dlp/yt-dlp/pull/557
+        'mweb': {
+            'INNERTUBE_CONTEXT': {
+                'client': {
+                    'clientName': 'MWEB',
+                    'clientVersion': '2.20241202.07.00',
+                    # mweb previously did not require PO Token with this UA
+                    'userAgent': 'Mozilla/5.0 (iPad; CPU OS 16_7_10 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1,gzip(gfe)',
+                },
+            },
+            'INNERTUBE_CONTEXT_CLIENT_NAME': 2,
+            'REQUIRE_PO_TOKEN': True,
+            'SUPPORTS_COOKIES': True,
+        },
+        'tv': {
+            'INNERTUBE_CONTEXT': {
+                'client': {
+                    'clientName': 'TVHTML5',
+                    'clientVersion': '7.20241201.18.00',
+                },
+            },
+            'INNERTUBE_CONTEXT_CLIENT_NAME': 7,
+            'SUPPORTS_COOKIES': True,
+        },
+    }
+
     def _login(self):
         """
         Attempt to log in to YouTube.
@@ -321,19 +351,24 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             '{0} {1} {2}'.format(time_now, self._SAPISID, origin).encode('utf-8')).hexdigest()
         return 'SAPISIDHASH {0}_{1}'.format(time_now, sapisidhash)
 
-    def _call_api(self, ep, query, video_id, fatal=True, headers=None):
+    def _call_api(self, ep, query, video_id, fatal=True, headers=None,
+                  note='Downloading API JSON'):
         data = self._DEFAULT_API_DATA.copy()
         data.update(query)
         real_headers = {'content-type': 'application/json'}
         if headers:
             real_headers.update(headers)
 
+        # was: 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'
+        api_key = self.get_param('youtube_innertube_key')
         return self._download_json(
             'https://www.youtube.com/youtubei/v1/%s' % ep, video_id=video_id,
-            note='Downloading API JSON', errnote='Unable to download API page',
+            note=note, errnote='Unable to download API page',
             data=json.dumps(data).encode('utf8'), fatal=fatal,
-            headers=real_headers,
-            query={'key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8'})
+            headers=real_headers, query=filter_dict({
+                'key': api_key,
+                'prettyPrint': 'false',
+            }))
 
     def _extract_yt_initial_data(self, video_id, webpage):
         return self._parse_json(
@@ -342,6 +377,22 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
                  self._YT_INITIAL_DATA_RE), webpage, 'yt initial data'),
             video_id)
 
+    def _extract_visitor_data(self, *args):
+        """
+        Extract visitorData from an API response or ytcfg
+
+        Appears to be used to track session state
+        """
+        visitor_data = self.get_param('youtube_visitor_data')
+        if visitor_data:
+            return visitor_data
+
+        return get_first(
+            args, (('VISITOR_DATA',
+                    ('INNERTUBE_CONTEXT', 'client', 'visitorData'),
+                    ('responseContext', 'visitorData')),
+                   T(compat_str)))
+
     def _extract_ytcfg(self, video_id, webpage):
         return self._parse_json(
             self._search_regex(
@@ -1957,6 +2008,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if sts:
                 pb_context['signatureTimestamp'] = sts
 
+            client = traverse_obj(self._INNERTUBE_CLIENTS, (
+                lambda _, v: not v.get('REQUIRE_PO_TOKEN')),
+                get_all=False)
+
             query = {
                 'playbackContext': {
                     'contentPlaybackContext': pb_context,
@@ -1964,30 +2019,39 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     'racyCheckOk': True,
                 },
                 'context': {
-                    'client': {
-                        'clientName': 'MWEB',
-                        'clientVersion': '2.20241202.07.00',
-                        'hl': 'en',
-                        'userAgent': 'Mozilla/5.0 (iPad; CPU OS 16_7_10 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1,gzip(gfe)',
-                        'timeZone': 'UTC',
-                        'utcOffsetMinutes': 0,
-                    },
+                    'client': merge_dicts(
+                        traverse_obj(client, ('INNERTUBE_CONTEXT', 'client')), {
+                            'hl': 'en',
+                            'timeZone': 'UTC',
+                            'utcOffsetMinutes': 0,
+                        }),
                 },
                 'videoId': video_id,
             }
-            headers = {
-                'X-YouTube-Client-Name': '2',
-                'X-YouTube-Client-Version': '2.20241202.07.00',
-                'Origin': origin,
+
+            headers = merge_dicts({
                 'Sec-Fetch-Mode': 'navigate',
-                'User-Agent': query['context']['client']['userAgent'],
-            }
+                'Origin': origin,
+                # 'X-Goog-Visitor-Id': self._extract_visitor_data(ytcfg) or '',
+            }, traverse_obj(client, {
+                'X-YouTube-Client-Name': 'INNERTUBE_CONTEXT_CLIENT_NAME',
+                'X-YouTube-Client-Version': (
+                    'INNERTUBE_CONTEXT', 'client', 'clientVersion'),
+                'User-Agent': (
+                    'INNERTUBE_CONTEXT', 'client', 'userAgent'),
+            }))
+
             auth = self._generate_sapisidhash_header(origin)
             if auth is not None:
                 headers['Authorization'] = auth
                 headers['X-Origin'] = origin
 
-            player_response = self._call_api('player', query, video_id, fatal=False, headers=headers)
+            player_response = self._call_api(
+                'player', query, video_id, fatal=False, headers=headers,
+                note=join_nonempty(
+                    'Downloading', traverse_obj(query, (
+                        'context', 'client', 'clientName')),
+                    'API JSON', delim=' '))
 
         def is_agegated(playability):
             if not isinstance(playability, dict):

From 55ad8a24cacee03a91fe70d8d48aa9a02cc0ab11 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 15 Jan 2025 03:22:56 +0000
Subject: [PATCH 807/860] [YouTube] Support `...
 /feeds/videos.xml?playlist_id={pl_id}`

---
 youtube_dl/extractor/youtube.py | 19 ++++++++++++++++---
 1 file changed, 16 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 1424277ac..f0406b357 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -3601,10 +3601,23 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
     def _real_extract(self, url):
         item_id = self._match_id(url)
         url = update_url(url, netloc='www.youtube.com')
-        # Handle both video/playlist URLs
         qs = parse_qs(url)
-        video_id = qs.get('v', [None])[0]
-        playlist_id = qs.get('list', [None])[0]
+
+        def qs_get(key, default=None):
+            return qs.get(key, [default])[-1]
+
+        # Go around for /feeds/videos.xml?playlist_id={pl_id}
+        if item_id == 'feeds' and '/feeds/videos.xml?' in url:
+            playlist_id = qs_get('playlist_id')
+            if playlist_id:
+                return self.url_result(
+                    update_url_query('https://www.youtube.com/playlist', {
+                        'list': playlist_id,
+                    }), ie=self.ie_key(), video_id=playlist_id)
+
+        # Handle both video/playlist URLs
+        video_id = qs_get('v')
+        playlist_id = qs_get('list')
         if video_id and playlist_id:
             if self._downloader.params.get('noplaylist'):
                 self.to_screen('Downloading just video %s because of --no-playlist' % video_id)

From b09442a2f4a8d255569abf0bb6b4867c53d0c2e9 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 19 Jan 2025 01:18:34 +0000
Subject: [PATCH 808/860] [YouTube] Also use ios client when is_live

---
 youtube_dl/extractor/youtube.py | 118 ++++++++++++++++++++++----------
 1 file changed, 81 insertions(+), 37 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index f0406b357..32e836d49 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -85,6 +85,22 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM)'
 
     _INNERTUBE_CLIENTS = {
+        'ios': {
+            'INNERTUBE_CONTEXT': {
+                'client': {
+                    'clientName': 'IOS',
+                    'clientVersion': '19.45.4',
+                    'deviceMake': 'Apple',
+                    'deviceModel': 'iPhone16,2',
+                    'userAgent': 'com.google.ios.youtube/19.45.4 (iPhone16,2; U; CPU iOS 18_1_0 like Mac OS X;)',
+                    'osName': 'iPhone',
+                    'osVersion': '18.1.0.22B83',
+                },
+            },
+            'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
+            'REQUIRE_JS_PLAYER': False,
+            'REQUIRE_PO_TOKEN': True,
+        },
         # mweb has 'ultralow' formats
         # See: https://github.com/yt-dlp/yt-dlp/pull/557
         'mweb': {
@@ -110,6 +126,17 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'INNERTUBE_CONTEXT_CLIENT_NAME': 7,
             'SUPPORTS_COOKIES': True,
         },
+        'web': {
+            'INNERTUBE_CONTEXT': {
+                'client': {
+                    'clientName': 'WEB',
+                    'clientVersion': '2.20241126.01.00',
+                },
+            },
+            'INNERTUBE_CONTEXT_CLIENT_NAME': 1,
+            'REQUIRE_PO_TOKEN': True,
+            'SUPPORTS_COOKIES': True,
+        },
     }
 
     def _login(self):
@@ -1995,6 +2022,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             player_response = self._extract_yt_initial_variable(
                 webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE,
                 video_id, 'initial player response')
+        is_live = traverse_obj(player_response, ('videoDetails', 'isLive'))
+
         if False and not player_response:
             player_response = self._call_api(
                 'player', {'videoId': video_id}, video_id)
@@ -2008,50 +2037,65 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             if sts:
                 pb_context['signatureTimestamp'] = sts
 
-            client = traverse_obj(self._INNERTUBE_CLIENTS, (
-                lambda _, v: not v.get('REQUIRE_PO_TOKEN')),
-                get_all=False)
+            client_names = traverse_obj(self._INNERTUBE_CLIENTS, (
+                T(dict.items), lambda _, k_v: not k_v[1].get('REQUIRE_PO_TOKEN'),
+                0))[:1]
 
-            query = {
-                'playbackContext': {
-                    'contentPlaybackContext': pb_context,
-                    'contentCheckOk': True,
-                    'racyCheckOk': True,
-                },
-                'context': {
-                    'client': merge_dicts(
-                        traverse_obj(client, ('INNERTUBE_CONTEXT', 'client')), {
-                            'hl': 'en',
-                            'timeZone': 'UTC',
-                            'utcOffsetMinutes': 0,
-                        }),
-                },
-                'videoId': video_id,
-            }
+            if is_live and 'ios' not in client_names:
+                client_names.append('ios')
 
-            headers = merge_dicts({
+            headers = {
                 'Sec-Fetch-Mode': 'navigate',
                 'Origin': origin,
                 # 'X-Goog-Visitor-Id': self._extract_visitor_data(ytcfg) or '',
-            }, traverse_obj(client, {
-                'X-YouTube-Client-Name': 'INNERTUBE_CONTEXT_CLIENT_NAME',
-                'X-YouTube-Client-Version': (
-                    'INNERTUBE_CONTEXT', 'client', 'clientVersion'),
-                'User-Agent': (
-                    'INNERTUBE_CONTEXT', 'client', 'userAgent'),
-            }))
-
+            }
             auth = self._generate_sapisidhash_header(origin)
             if auth is not None:
                 headers['Authorization'] = auth
                 headers['X-Origin'] = origin
 
-            player_response = self._call_api(
-                'player', query, video_id, fatal=False, headers=headers,
-                note=join_nonempty(
-                    'Downloading', traverse_obj(query, (
-                        'context', 'client', 'clientName')),
-                    'API JSON', delim=' '))
+            for client in traverse_obj(self._INNERTUBE_CLIENTS, (client_names, T(dict))):
+
+                query = {
+                    'playbackContext': {
+                        'contentPlaybackContext': pb_context,
+                        'contentCheckOk': True,
+                        'racyCheckOk': True,
+                    },
+                    'context': {
+                        'client': merge_dicts(
+                            traverse_obj(client, ('INNERTUBE_CONTEXT', 'client')), {
+                                'hl': 'en',
+                                'timeZone': 'UTC',
+                                'utcOffsetMinutes': 0,
+                            }),
+                    },
+                    'videoId': video_id,
+                }
+
+                api_headers = merge_dicts(headers, traverse_obj(client, {
+                    'X-YouTube-Client-Name': 'INNERTUBE_CONTEXT_CLIENT_NAME',
+                    'X-YouTube-Client-Version': (
+                        'INNERTUBE_CONTEXT', 'client', 'clientVersion'),
+                    'User-Agent': (
+                        'INNERTUBE_CONTEXT', 'client', 'userAgent'),
+                }))
+
+                api_player_response = self._call_api(
+                    'player', query, video_id, fatal=False, headers=api_headers,
+                    note=join_nonempty(
+                        'Downloading', traverse_obj(query, (
+                            'context', 'client', 'clientName')),
+                        'API JSON', delim=' '))
+
+                hls = [
+                    traverse_obj(
+                        resp, ('streamingData', 'hlsManifestUrl', T(url_or_none)))
+                    for resp in (player_response, api_player_response)]
+                if not hls[0] and hls[1]:
+                    player_response['streamingData']['hlsManifestUrl'] = hls[1]
+                else:
+                    player_response.update(api_player_response or {})
 
         def is_agegated(playability):
             if not isinstance(playability, dict):
@@ -2194,6 +2238,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         itag_qualities = {}
         q = qualities(['tiny', 'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres'])
         CHUNK_SIZE = 10 << 20
+        is_live = video_details.get('isLive')
 
         streaming_data = player_response.get('streamingData') or {}
         streaming_formats = streaming_data.get('formats') or []
@@ -2338,7 +2383,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         hls_manifest_url = streaming_data.get('hlsManifestUrl')
         if hls_manifest_url:
             for f in self._extract_m3u8_formats(
-                    hls_manifest_url, video_id, 'mp4', fatal=False):
+                    hls_manifest_url, video_id, 'mp4',
+                    entry_protocol='m3u8_native', live=is_live, fatal=False):
                 if process_manifest_format(
                         f, 'hls', None, self._search_regex(
                             r'/itag/(\d+)', f['url'], 'itag', default=None)):
@@ -2444,8 +2490,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 # Strictly de-prioritize damaged formats
                 f['preference'] = -10
 
-        is_live = video_details.get('isLive')
-
         owner_profile_url = self._yt_urljoin(self._extract_author_var(
             webpage, 'url', videodetails=video_details, metadata=microformat))
 

From 63fb0fc4159397618b12fa115f957b9ba70f3f88 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 20 Jan 2025 13:23:54 +0000
Subject: [PATCH 809/860] [YouTube] Retain .videoDetails members from all
 player responses

---
 youtube_dl/extractor/youtube.py | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 32e836d49..edaae5bd3 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2095,7 +2095,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 if not hls[0] and hls[1]:
                     player_response['streamingData']['hlsManifestUrl'] = hls[1]
                 else:
+                    video_details = merge_dicts(*traverse_obj(
+                        (player_response, api_player_response),
+                        (Ellipsis, 'videoDetails', T(dict))))
                     player_response.update(api_player_response or {})
+                    player_response['videoDetails'] = video_details
 
         def is_agegated(playability):
             if not isinstance(playability, dict):

From 5975d7bb96095fae7c35e7cfcd819255a5b57087 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 22 Jan 2025 06:52:40 +0000
Subject: [PATCH 810/860] [YouTube] Use X-Goog-Visitor-Id * required with tv
 player client * resolves #33030

---
 youtube_dl/extractor/youtube.py | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index edaae5bd3..c93a2a1f9 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -142,6 +142,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
     def _login(self):
         """
         Attempt to log in to YouTube.
+
         True is returned if successful or skipped.
         False is returned if login failed.
 
@@ -2040,6 +2041,11 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             client_names = traverse_obj(self._INNERTUBE_CLIENTS, (
                 T(dict.items), lambda _, k_v: not k_v[1].get('REQUIRE_PO_TOKEN'),
                 0))[:1]
+            if 'web' not in client_names:
+                # webpage links won't download: ignore links and playability
+                player_response = filter_dict(
+                    player_response or {},
+                    lambda k, _: k not in ('streamingData', 'playabilityStatus'))
 
             if is_live and 'ios' not in client_names:
                 client_names.append('ios')
@@ -2047,7 +2053,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             headers = {
                 'Sec-Fetch-Mode': 'navigate',
                 'Origin': origin,
-                # 'X-Goog-Visitor-Id': self._extract_visitor_data(ytcfg) or '',
+                'X-Goog-Visitor-Id': self._extract_visitor_data(ytcfg) or '',
             }
             auth = self._generate_sapisidhash_header(origin)
             if auth is not None:
@@ -2059,9 +2065,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 query = {
                     'playbackContext': {
                         'contentPlaybackContext': pb_context,
-                        'contentCheckOk': True,
-                        'racyCheckOk': True,
                     },
+                    'contentCheckOk': True,
+                    'racyCheckOk': True,
                     'context': {
                         'client': merge_dicts(
                             traverse_obj(client, ('INNERTUBE_CONTEXT', 'client')), {
@@ -2088,11 +2094,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             'context', 'client', 'clientName')),
                         'API JSON', delim=' '))
 
-                hls = [
-                    traverse_obj(
-                        resp, ('streamingData', 'hlsManifestUrl', T(url_or_none)))
-                    for resp in (player_response, api_player_response)]
-                if not hls[0] and hls[1]:
+                hls = traverse_obj(
+                    (player_response, api_player_response),
+                    (Ellipsis, 'streamingData', 'hlsManifestUrl', T(url_or_none)))
+                if len(hls) == 2 and not hls[0] and hls[1]:
                     player_response['streamingData']['hlsManifestUrl'] = hls[1]
                 else:
                     video_details = merge_dicts(*traverse_obj(
@@ -3467,7 +3472,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             if not continuation:
                 break
             if visitor_data:
-                headers['x-goog-visitor-id'] = visitor_data
+                headers['X-Goog-Visitor-Id'] = visitor_data
             data['continuation'] = continuation['continuation']
             data['clickTracking'] = {
                 'clickTrackingParams': continuation['itct'],

From 26b6f15d14e30dc3349abe2933edb051b46be4b8 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 3 Feb 2025 18:10:15 +0000
Subject: [PATCH 811/860] [compat] Make casefold private * if required, not
 supported: `from youtube_dl.casefold import _casefold as casefold`

---
 youtube_dl/casefold.py | 12 ++++--------
 youtube_dl/compat.py   |  2 +-
 2 files changed, 5 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/casefold.py b/youtube_dl/casefold.py
index ad9c66f8e..712b2e7fa 100644
--- a/youtube_dl/casefold.py
+++ b/youtube_dl/casefold.py
@@ -10,9 +10,10 @@ from .compat import (
 # https://github.com/unicode-org/icu/blob/main/icu4c/source/data/unidata/CaseFolding.txt
 # In case newly foldable Unicode characters are defined, paste the new version
 # of the text inside the ''' marks.
-# The text is expected to have only blank lines andlines with 1st character #,
+# The text is expected to have only blank lines and lines with 1st character #,
 # all ignored, and fold definitions like this:
-# `from_hex_code; space_separated_to_hex_code_list; comment`
+# `from_hex_code; status; space_separated_to_hex_code_list; comment`
+# Only `status` C/F are used.
 
 _map_str = '''
 # CaseFolding-15.0.0.txt
@@ -1657,11 +1658,6 @@ _map = dict(
 del _map_str
 
 
-def casefold(s):
+def _casefold(s):
     assert isinstance(s, compat_str)
     return ''.join((_map.get(c, c) for c in s))
-
-
-__all__ = [
-    'casefold',
-]
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index ed1a33cf2..c621f7476 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -44,7 +44,7 @@ try:
     compat_str.casefold
     compat_casefold = lambda s: s.casefold()
 except AttributeError:
-    from .casefold import casefold as compat_casefold
+    from .casefold import _casefold as compat_casefold
 
 try:
     import collections.abc as compat_collections_abc

From 711e72c292327674c4a0593fdbb83d6347738ec9 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Thu, 6 Feb 2025 21:09:00 +0000
Subject: [PATCH 812/860] [JSInterp] Fix bit-shift coercion for player 9c6dfc4a

---
 test/test_jsinterp.py          |  4 ++++
 test/test_youtube_signature.py |  4 ++++
 youtube_dl/compat.py           |  5 +++++
 youtube_dl/jsinterp.py         | 27 +++++++++++++++++++++------
 4 files changed, 34 insertions(+), 6 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 12e7b9b94..6c34bc896 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -459,6 +459,10 @@ class TestJSInterpreter(unittest.TestCase):
         self._test('function f(){return undefined >> 5}', 0)
         self._test('function f(){return 42 << NaN}', 42)
         self._test('function f(){return 42 << Infinity}', 42)
+        self._test('function f(){return 0.0 << null}', 0)
+        self._test('function f(){return NaN << 42}', 0)
+        self._test('function f(){return "21.9" << 1}', 42)
+        self._test('function f(){return 21 << 4294967297}', 42)
 
     def test_negative(self):
         self._test('function f(){return 2    *    -2.0    ;}', -4)
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index fcbc9d7a8..67ef75fde 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -219,6 +219,10 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/2f1832d2/player_ias.vflset/en_US/base.js',
         'YWt1qdbe8SAfkoPHW5d', 'RrRjWQOJmBiP',
     ),
+    (
+        'https://www.youtube.com/s/player/9c6dfc4a/player_ias.vflset/en_US/base.js',
+        'jbu7ylIosQHyJyJV', 'uwI0ESiynAmhNg',
+    ),
 ]
 
 
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index c621f7476..26b655fb6 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -3116,17 +3116,21 @@ else:
     compat_kwargs = lambda kwargs: kwargs
 
 
+# compat_numeric_types
 try:
     compat_numeric_types = (int, float, long, complex)
 except NameError:  # Python 3
     compat_numeric_types = (int, float, complex)
 
 
+# compat_integer_types
 try:
     compat_integer_types = (int, long)
 except NameError:  # Python 3
     compat_integer_types = (int, )
 
+# compat_int
+compat_int = compat_integer_types[-1]
 
 if sys.version_info < (2, 7):
     def compat_socket_create_connection(address, timeout, source_address=None):
@@ -3532,6 +3536,7 @@ __all__ = [
     'compat_http_client',
     'compat_http_server',
     'compat_input',
+    'compat_int',
     'compat_integer_types',
     'compat_itertools_count',
     'compat_itertools_zip_longest',
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 7835187f5..2859bc734 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -24,6 +24,8 @@ from .compat import (
     compat_collections_chain_map as ChainMap,
     compat_contextlib_suppress,
     compat_filter as filter,
+    compat_int,
+    compat_integer_types,
     compat_itertools_zip_longest as zip_longest,
     compat_map as map,
     compat_numeric_types,
@@ -70,14 +72,27 @@ class JS_Undefined(object):
     pass
 
 
-def _js_bit_op(op):
+def _js_bit_op(op, is_shift=False):
 
-    def zeroise(x):
-        return 0 if x in (None, JS_Undefined, _NaN, _Infinity) else x
+    def zeroise(x, is_shift_arg=False):
+        if isinstance(x, compat_integer_types):
+            return (x % 32) if is_shift_arg else (x & 0xffffffff)
+        try:
+            x = float(x)
+            if is_shift_arg:
+                x = int(x % 32)
+            elif x < 0:
+                x = -compat_int(-x % 0xffffffff)
+            else:
+                x = compat_int(x % 0xffffffff)
+        except (ValueError, TypeError):
+            # also here for int(NaN), including float('inf') % 32
+            x = 0
+        return x
 
     @wraps_op(op)
     def wrapped(a, b):
-        return op(zeroise(a), zeroise(b)) & 0xffffffff
+        return op(zeroise(a), zeroise(b, is_shift)) & 0xffffffff
 
     return wrapped
 
@@ -253,8 +268,8 @@ def _js_typeof(expr):
 # avoid dict to maintain order
 # definition None => Defined in JSInterpreter._operator
 _OPERATORS = (
-    ('>>', _js_bit_op(operator.rshift)),
-    ('<<', _js_bit_op(operator.lshift)),
+    ('>>', _js_bit_op(operator.rshift, True)),
+    ('<<', _js_bit_op(operator.lshift, True)),
     ('+', _js_add),
     ('-', _js_arith_op(operator.sub)),
     ('*', _js_arith_op(operator.mul)),

From 91b1569f68471d685382b738806b2e07d8f52707 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 28 Feb 2025 00:02:10 +0000
Subject: [PATCH 813/860] [YouTube] Fix channel playlist extraction (#33074)

* [YouTube] Extract playlist items from LOCKUP_VIEW_MODEL_...
* resolves #33073
* thx seproDev (yt-dlp/yt-dlp#11615)

Co-authored-by: sepro <sepro@sepr0.com>
---
 youtube_dl/extractor/youtube.py | 49 +++++++++++++++++++++++++++++++--
 1 file changed, 47 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index c93a2a1f9..cc84a193a 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -27,6 +27,7 @@ from ..compat import (
 )
 from ..jsinterp import JSInterpreter
 from ..utils import (
+    bug_reports_message,
     clean_html,
     dict_get,
     error_to_compat_str,
@@ -65,6 +66,7 @@ from ..utils import (
     url_or_none,
     urlencode_postdata,
     urljoin,
+    variadic,
 )
 
 
@@ -460,6 +462,26 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'uploader': uploader,
         }
 
+    @staticmethod
+    def _extract_thumbnails(data, *path_list, **kw_final_key):
+        """
+        Extract thumbnails from thumbnails dict
+        @param path_list: path list to level that contains 'thumbnails' key
+        """
+        final_key = kw_final_key.get('final_key', 'thumbnails')
+
+        return traverse_obj(data, ((
+            tuple(variadic(path) + (final_key, Ellipsis)
+                  for path in path_list or [()])), {
+            'url': ('url', T(url_or_none),
+                    # Sometimes youtube gives a wrong thumbnail URL. See:
+                    # https://github.com/yt-dlp/yt-dlp/issues/233
+                    # https://github.com/ytdl-org/youtube-dl/issues/28023
+                    T(lambda u: update_url(u, query=None) if u and 'maxresdefault' in u else u)),
+            'height': ('height', T(int_or_none)),
+            'width': ('width', T(int_or_none)),
+        }, T(lambda t: t if t.get('url') else None)))
+
     def _search_results(self, query, params):
         data = {
             'context': {
@@ -3183,8 +3205,12 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             expected_type=txt_or_none)
 
     def _grid_entries(self, grid_renderer):
-        for item in grid_renderer['items']:
-            if not isinstance(item, dict):
+        for item in traverse_obj(grid_renderer, ('items', Ellipsis, T(dict))):
+            lockup_view_model = traverse_obj(item, ('lockupViewModel', T(dict)))
+            if lockup_view_model:
+                entry = self._extract_lockup_view_model(lockup_view_model)
+                if entry:
+                    yield entry
                 continue
             renderer = self._extract_grid_item_renderer(item)
             if not isinstance(renderer, dict):
@@ -3268,6 +3294,25 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                 continue
             yield self._extract_video(renderer)
 
+    def _extract_lockup_view_model(self, view_model):
+        content_id = view_model.get('contentId')
+        if not content_id:
+            return
+        content_type = view_model.get('contentType')
+        if content_type not in ('LOCKUP_CONTENT_TYPE_PLAYLIST', 'LOCKUP_CONTENT_TYPE_PODCAST'):
+            self.report_warning(
+                'Unsupported lockup view model content type "{0}"{1}'.format(content_type, bug_reports_message()), only_once=True)
+            return
+        return merge_dicts(self.url_result(
+            update_url_query('https://www.youtube.com/playlist', {'list': content_id}),
+            ie=YoutubeTabIE, video_id=content_id), {
+                'title': traverse_obj(view_model, (
+                    'metadata', 'lockupMetadataViewModel', 'title', 'content', T(compat_str))),
+                'thumbnails': self._extract_thumbnails(view_model, (
+                    'contentImage', 'collectionThumbnailViewModel', 'primaryThumbnail',
+                    'thumbnailViewModel', 'image'), final_key='sources'),
+        })
+
     def _video_entry(self, video_renderer):
         video_id = video_renderer.get('videoId')
         if video_id:

From 673277e510ebd996b62a2fcc76169bf3cce29910 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 28 Feb 2025 01:02:20 +0000
Subject: [PATCH 814/860] [YouTube] Fix 91b1569

---
 youtube_dl/extractor/youtube.py | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index cc84a193a..5f8c08201 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -3305,7 +3305,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             return
         return merge_dicts(self.url_result(
             update_url_query('https://www.youtube.com/playlist', {'list': content_id}),
-            ie=YoutubeTabIE, video_id=content_id), {
+            ie=YoutubeTabIE.ie_key(), video_id=content_id), {
                 'title': traverse_obj(view_model, (
                     'metadata', 'lockupMetadataViewModel', 'title', 'content', T(compat_str))),
                 'thumbnails': self._extract_thumbnails(view_model, (

From cecaa18b80e33323193915ef9fbd2f68d94d7bce Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 7 Mar 2025 23:03:17 +0000
Subject: [PATCH 815/860] [compat] Clean-up * make workaround_optparse_bug9161
 private * add comments * avoid leaving test objects behind

---
 youtube_dl/__init__.py |   4 +-
 youtube_dl/compat.py   | 172 ++++++++++++++++++++++++++++++-----------
 2 files changed, 129 insertions(+), 47 deletions(-)

diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py
index 06bdfb689..3c1272e7b 100644
--- a/youtube_dl/__init__.py
+++ b/youtube_dl/__init__.py
@@ -18,7 +18,7 @@ from .compat import (
     compat_getpass,
     compat_register_utf8,
     compat_shlex_split,
-    workaround_optparse_bug9161,
+    _workaround_optparse_bug9161,
 )
 from .utils import (
     _UnsafeExtensionError,
@@ -50,7 +50,7 @@ def _real_main(argv=None):
     # Compatibility fix for Windows
     compat_register_utf8()
 
-    workaround_optparse_bug9161()
+    _workaround_optparse_bug9161()
 
     setproctitle('youtube-dl')
 
diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 26b655fb6..e617dd511 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -16,7 +16,6 @@ import os
 import platform
 import re
 import shlex
-import shutil
 import socket
 import struct
 import subprocess
@@ -24,11 +23,15 @@ import sys
 import types
 import xml.etree.ElementTree
 
+_IDENTITY = lambda x: x
+
 # naming convention
 # 'compat_' + Python3_name.replace('.', '_')
 # other aliases exist for convenience and/or legacy
+# wrap disposable test values in type() to reclaim storage
 
-# deal with critical unicode/str things first
+# deal with critical unicode/str things first:
+# compat_str, compat_basestring, compat_chr
 try:
     # Python 2
     compat_str, compat_basestring, compat_chr = (
@@ -39,18 +42,23 @@ except NameError:
         str, (str, bytes), chr
     )
 
-# casefold
+
+# compat_casefold
 try:
     compat_str.casefold
     compat_casefold = lambda s: s.casefold()
 except AttributeError:
     from .casefold import _casefold as compat_casefold
 
+
+# compat_collections_abc
 try:
     import collections.abc as compat_collections_abc
 except ImportError:
     import collections as compat_collections_abc
 
+
+# compat_urllib_request
 try:
     import urllib.request as compat_urllib_request
 except ImportError:  # Python 2
@@ -79,11 +87,15 @@ except TypeError:
     _add_init_method_arg(compat_urllib_request.Request)
     del _add_init_method_arg
 
+
+# compat_urllib_error
 try:
     import urllib.error as compat_urllib_error
 except ImportError:  # Python 2
     import urllib2 as compat_urllib_error
 
+
+# compat_urllib_parse
 try:
     import urllib.parse as compat_urllib_parse
 except ImportError:  # Python 2
@@ -98,17 +110,23 @@ except ImportError:  # Python 2
 compat_urlparse = compat_urllib_parse
 compat_urllib_parse_urlparse = compat_urllib_parse.urlparse
 
+
+# compat_urllib_response
 try:
     import urllib.response as compat_urllib_response
 except ImportError:  # Python 2
     import urllib as compat_urllib_response
 
+
+# compat_urllib_response.addinfourl
 try:
     compat_urllib_response.addinfourl.status
 except AttributeError:
     # .getcode() is deprecated in Py 3.
     compat_urllib_response.addinfourl.status = property(lambda self: self.getcode())
 
+
+# compat_http_cookiejar
 try:
     import http.cookiejar as compat_cookiejar
 except ImportError:  # Python 2
@@ -127,12 +145,16 @@ else:
     compat_cookiejar_Cookie = compat_cookiejar.Cookie
 compat_http_cookiejar_Cookie = compat_cookiejar_Cookie
 
+
+# compat_http_cookies
 try:
     import http.cookies as compat_cookies
 except ImportError:  # Python 2
     import Cookie as compat_cookies
 compat_http_cookies = compat_cookies
 
+
+# compat_http_cookies_SimpleCookie
 if sys.version_info[0] == 2 or sys.version_info < (3, 3):
     class compat_cookies_SimpleCookie(compat_cookies.SimpleCookie):
         def load(self, rawdata):
@@ -155,11 +177,15 @@ else:
     compat_cookies_SimpleCookie = compat_cookies.SimpleCookie
 compat_http_cookies_SimpleCookie = compat_cookies_SimpleCookie
 
+
+# compat_html_entities, probably useless now
 try:
     import html.entities as compat_html_entities
 except ImportError:  # Python 2
     import htmlentitydefs as compat_html_entities
 
+
+# compat_html_entities_html5
 try:  # Python >= 3.3
     compat_html_entities_html5 = compat_html_entities.html5
 except AttributeError:
@@ -2408,18 +2434,24 @@ except AttributeError:
     # Py < 3.1
     compat_http_client.HTTPResponse.getcode = lambda self: self.status
 
+
+# compat_urllib_HTTPError
 try:
     from urllib.error import HTTPError as compat_HTTPError
 except ImportError:  # Python 2
     from urllib2 import HTTPError as compat_HTTPError
 compat_urllib_HTTPError = compat_HTTPError
 
+
+# compat_urllib_request_urlretrieve
 try:
     from urllib.request import urlretrieve as compat_urlretrieve
 except ImportError:  # Python 2
     from urllib import urlretrieve as compat_urlretrieve
 compat_urllib_request_urlretrieve = compat_urlretrieve
 
+
+# compat_html_parser_HTMLParser, compat_html_parser_HTMLParseError
 try:
     from HTMLParser import (
         HTMLParser as compat_HTMLParser,
@@ -2432,22 +2464,33 @@ except ImportError:  # Python 3
         # HTMLParseError was deprecated in Python 3.3 and removed in
         # Python 3.5. Introducing dummy exception for Python >3.5 for compatible
         # and uniform cross-version exception handling
+
         class compat_HTMLParseError(Exception):
             pass
+
 compat_html_parser_HTMLParser = compat_HTMLParser
 compat_html_parser_HTMLParseError = compat_HTMLParseError
 
+
+# compat_subprocess_get_DEVNULL
 try:
     _DEVNULL = subprocess.DEVNULL
     compat_subprocess_get_DEVNULL = lambda: _DEVNULL
 except AttributeError:
     compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w')
 
+
+# compat_http_server
 try:
     import http.server as compat_http_server
 except ImportError:
     import BaseHTTPServer as compat_http_server
 
+
+# compat_urllib_parse_unquote_to_bytes,
+# compat_urllib_parse_unquote, compat_urllib_parse_unquote_plus,
+# compat_urllib_parse_urlencode,
+# compat_urllib_parse_parse_qs
 try:
     from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes
     from urllib.parse import unquote as compat_urllib_parse_unquote
@@ -2598,6 +2641,8 @@ except ImportError:  # Python 2
 
 compat_urllib_parse_parse_qs = compat_parse_qs
 
+
+# compat_urllib_request_DataHandler
 try:
     from urllib.request import DataHandler as compat_urllib_request_DataHandler
 except ImportError:  # Python < 3.4
@@ -2632,16 +2677,20 @@ except ImportError:  # Python < 3.4
 
             return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url)
 
+
+# compat_xml_etree_ElementTree_ParseError
 try:
     from xml.etree.ElementTree import ParseError as compat_xml_parse_error
 except ImportError:  # Python 2.6
     from xml.parsers.expat import ExpatError as compat_xml_parse_error
 compat_xml_etree_ElementTree_ParseError = compat_xml_parse_error
 
-etree = xml.etree.ElementTree
+
+# compat_xml_etree_ElementTree_Element
+_etree = xml.etree.ElementTree
 
 
-class _TreeBuilder(etree.TreeBuilder):
+class _TreeBuilder(_etree.TreeBuilder):
     def doctype(self, name, pubid, system):
         pass
 
@@ -2650,7 +2699,7 @@ try:
     # xml.etree.ElementTree.Element is a method in Python <=2.6 and
     # the following will crash with:
     #  TypeError: isinstance() arg 2 must be a class, type, or tuple of classes and types
-    isinstance(None, etree.Element)
+    isinstance(None, _etree.Element)
     from xml.etree.ElementTree import Element as compat_etree_Element
 except TypeError:  # Python <=2.6
     from xml.etree.ElementTree import _ElementInterface as compat_etree_Element
@@ -2658,12 +2707,12 @@ compat_xml_etree_ElementTree_Element = compat_etree_Element
 
 if sys.version_info[0] >= 3:
     def compat_etree_fromstring(text):
-        return etree.XML(text, parser=etree.XMLParser(target=_TreeBuilder()))
+        return _etree.XML(text, parser=_etree.XMLParser(target=_TreeBuilder()))
 else:
     # python 2.x tries to encode unicode strings with ascii (see the
     # XMLParser._fixtext method)
     try:
-        _etree_iter = etree.Element.iter
+        _etree_iter = _etree.Element.iter
     except AttributeError:  # Python <=2.6
         def _etree_iter(root):
             for el in root.findall('*'):
@@ -2675,27 +2724,29 @@ else:
     # 2.7 source
     def _XML(text, parser=None):
         if not parser:
-            parser = etree.XMLParser(target=_TreeBuilder())
+            parser = _etree.XMLParser(target=_TreeBuilder())
         parser.feed(text)
         return parser.close()
 
     def _element_factory(*args, **kwargs):
-        el = etree.Element(*args, **kwargs)
+        el = _etree.Element(*args, **kwargs)
         for k, v in el.items():
             if isinstance(v, bytes):
                 el.set(k, v.decode('utf-8'))
         return el
 
     def compat_etree_fromstring(text):
-        doc = _XML(text, parser=etree.XMLParser(target=_TreeBuilder(element_factory=_element_factory)))
+        doc = _XML(text, parser=_etree.XMLParser(target=_TreeBuilder(element_factory=_element_factory)))
         for el in _etree_iter(doc):
             if el.text is not None and isinstance(el.text, bytes):
                 el.text = el.text.decode('utf-8')
         return doc
 
-if hasattr(etree, 'register_namespace'):
-    compat_etree_register_namespace = etree.register_namespace
-else:
+
+# compat_xml_etree_register_namespace
+try:
+    compat_etree_register_namespace = _etree.register_namespace
+except AttributeError:
     def compat_etree_register_namespace(prefix, uri):
         """Register a namespace prefix.
         The registry is global, and any existing mapping for either the
@@ -2704,14 +2755,16 @@ else:
         attributes in this namespace will be serialized with prefix if possible.
         ValueError is raised if prefix is reserved or is invalid.
         """
-        if re.match(r"ns\d+$", prefix):
-            raise ValueError("Prefix format reserved for internal use")
-        for k, v in list(etree._namespace_map.items()):
+        if re.match(r'ns\d+$', prefix):
+            raise ValueError('Prefix format reserved for internal use')
+        for k, v in list(_etree._namespace_map.items()):
             if k == uri or v == prefix:
-                del etree._namespace_map[k]
-        etree._namespace_map[uri] = prefix
+                del _etree._namespace_map[k]
+        _etree._namespace_map[uri] = prefix
 compat_xml_etree_register_namespace = compat_etree_register_namespace
 
+
+# compat_xpath, compat_etree_iterfind
 if sys.version_info < (2, 7):
     # Here comes the crazy part: In 2.6, if the xpath is a unicode,
     # .//node does not match if a node is a direct child of . !
@@ -2898,7 +2951,6 @@ if sys.version_info < (2, 7):
         def __init__(self, root):
             self.root = root
 
-    ##
     # Generate all matching objects.
 
     def compat_etree_iterfind(elem, path, namespaces=None):
@@ -2933,13 +2985,15 @@ if sys.version_info < (2, 7):
 
 
 else:
-    compat_xpath = lambda xpath: xpath
     compat_etree_iterfind = lambda element, match: element.iterfind(match)
+    compat_xpath = _IDENTITY
 
 
+# compat_os_name
 compat_os_name = os._name if os.name == 'java' else os.name
 
 
+# compat_shlex_quote
 if compat_os_name == 'nt':
     def compat_shlex_quote(s):
         return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"')
@@ -2954,6 +3008,7 @@ else:
                 return "'" + s.replace("'", "'\"'\"'") + "'"
 
 
+# compat_shlex.split
 try:
     args = shlex.split('中文')
     assert (isinstance(args, list)
@@ -2969,6 +3024,7 @@ except (AssertionError, UnicodeEncodeError):
         return list(map(lambda s: s.decode('utf-8'), shlex.split(s, comments, posix)))
 
 
+# compat_ord
 def compat_ord(c):
     if isinstance(c, int):
         return c
@@ -2976,6 +3032,7 @@ def compat_ord(c):
         return ord(c)
 
 
+# compat_getenv, compat_os_path_expanduser, compat_setenv
 if sys.version_info >= (3, 0):
     compat_getenv = os.getenv
     compat_expanduser = os.path.expanduser
@@ -3063,6 +3120,7 @@ else:
 compat_os_path_expanduser = compat_expanduser
 
 
+# compat_os_path_realpath
 if compat_os_name == 'nt' and sys.version_info < (3, 8):
     # os.path.realpath on Windows does not follow symbolic links
     # prior to Python 3.8 (see https://bugs.python.org/issue9949)
@@ -3076,6 +3134,7 @@ else:
 compat_os_path_realpath = compat_realpath
 
 
+# compat_print
 if sys.version_info < (3, 0):
     def compat_print(s):
         from .utils import preferredencoding
@@ -3086,6 +3145,7 @@ else:
         print(s)
 
 
+# compat_getpass_getpass
 if sys.version_info < (3, 0) and sys.platform == 'win32':
     def compat_getpass(prompt, *args, **kwargs):
         if isinstance(prompt, compat_str):
@@ -3098,22 +3158,22 @@ else:
 compat_getpass_getpass = compat_getpass
 
 
+# compat_input
 try:
     compat_input = raw_input
 except NameError:  # Python 3
     compat_input = input
 
 
+# compat_kwargs
 # Python < 2.6.5 require kwargs to be bytes
 try:
-    def _testfunc(x):
-        pass
-    _testfunc(**{'x': 0})
+    (lambda x: x)(**{'x': 0})
 except TypeError:
     def compat_kwargs(kwargs):
         return dict((bytes(k), v) for k, v in kwargs.items())
 else:
-    compat_kwargs = lambda kwargs: kwargs
+    compat_kwargs = _IDENTITY
 
 
 # compat_numeric_types
@@ -3132,6 +3192,8 @@ except NameError:  # Python 3
 # compat_int
 compat_int = compat_integer_types[-1]
 
+
+# compat_socket_create_connection
 if sys.version_info < (2, 7):
     def compat_socket_create_connection(address, timeout, source_address=None):
         host, port = address
@@ -3158,6 +3220,7 @@ else:
     compat_socket_create_connection = socket.create_connection
 
 
+# compat_contextlib_suppress
 try:
     from contextlib import suppress as compat_contextlib_suppress
 except ImportError:
@@ -3200,12 +3263,12 @@ except AttributeError:
                         # repeated .close() is OK, but just in case
                         with compat_contextlib_suppress(EnvironmentError):
                             f.close()
-                popen.wait()
+            popen.wait()
 
 
 # Fix https://github.com/ytdl-org/youtube-dl/issues/4223
 # See http://bugs.python.org/issue9161 for what is broken
-def workaround_optparse_bug9161():
+def _workaround_optparse_bug9161():
     op = optparse.OptionParser()
     og = optparse.OptionGroup(op, 'foo')
     try:
@@ -3224,9 +3287,10 @@ def workaround_optparse_bug9161():
         optparse.OptionGroup.add_option = _compat_add_option
 
 
-if hasattr(shutil, 'get_terminal_size'):  # Python >= 3.3
-    compat_get_terminal_size = shutil.get_terminal_size
-else:
+# compat_shutil_get_terminal_size
+try:
+    from shutil import get_terminal_size as compat_get_terminal_size  # Python >= 3.3
+except ImportError:
     _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines'])
 
     def compat_get_terminal_size(fallback=(80, 24)):
@@ -3256,27 +3320,33 @@ else:
                 columns = _columns
             if lines is None or lines <= 0:
                 lines = _lines
+
         return _terminal_size(columns, lines)
 
+compat_shutil_get_terminal_size = compat_get_terminal_size
 
+
+# compat_itertools_count
 try:
-    itertools.count(start=0, step=1)
+    type(itertools.count(start=0, step=1))
     compat_itertools_count = itertools.count
-except TypeError:  # Python 2.6
+except TypeError:  # Python 2.6 lacks step
     def compat_itertools_count(start=0, step=1):
         while True:
             yield start
             start += step
 
 
+# compat_tokenize_tokenize
 if sys.version_info >= (3, 0):
     from tokenize import tokenize as compat_tokenize_tokenize
 else:
     from tokenize import generate_tokens as compat_tokenize_tokenize
 
 
+# compat_struct_pack, compat_struct_unpack, compat_Struct
 try:
-    struct.pack('!I', 0)
+    type(struct.pack('!I', 0))
 except TypeError:
     # In Python 2.6 and 2.7.x < 2.7.7, struct requires a bytes argument
     # See https://bugs.python.org/issue19099
@@ -3308,8 +3378,10 @@ else:
         compat_Struct = struct.Struct
 
 
-# compat_map/filter() returning an iterator, supposedly the
-# same versioning as for zip below
+# builtins returning an iterator
+
+# compat_map, compat_filter
+# supposedly the same versioning as for zip below
 try:
     from future_builtins import map as compat_map
 except ImportError:
@@ -3326,6 +3398,7 @@ except ImportError:
     except ImportError:
         compat_filter = filter
 
+# compat_zip
 try:
     from future_builtins import zip as compat_zip
 except ImportError:  # not 2.6+ or is 3.x
@@ -3335,6 +3408,7 @@ except ImportError:  # not 2.6+ or is 3.x
         compat_zip = zip
 
 
+# compat_itertools_zip_longest
 # method renamed between Py2/3
 try:
     from itertools import zip_longest as compat_itertools_zip_longest
@@ -3342,7 +3416,8 @@ except ImportError:
     from itertools import izip_longest as compat_itertools_zip_longest
 
 
-# new class in collections
+# compat_collections_chain_map
+# collections.ChainMap: new class
 try:
     from collections import ChainMap as compat_collections_chain_map
     # Py3.3's ChainMap is deficient
@@ -3405,12 +3480,14 @@ except ImportError:
             return compat_collections_chain_map(*(self.maps[1:]))
 
 
+# compat_re_Pattern, compat_re_Match
 # Pythons disagree on the type of a pattern (RegexObject, _sre.SRE_Pattern, Pattern, ...?)
 compat_re_Pattern = type(re.compile(''))
 # and on the type of a match
 compat_re_Match = type(re.match('a', 'a'))
 
 
+# compat_base64_b64decode
 if sys.version_info < (3, 3):
     def compat_b64decode(s, *args, **kwargs):
         if isinstance(s, compat_str):
@@ -3422,6 +3499,7 @@ else:
 compat_base64_b64decode = compat_b64decode
 
 
+# compat_ctypes_WINFUNCTYPE
 if platform.python_implementation() == 'PyPy' and sys.pypy_version_info < (5, 4, 0):
     # PyPy2 prior to version 5.4.0 expects byte strings as Windows function
     # names, see the original PyPy issue [1] and the youtube-dl one [2].
@@ -3440,6 +3518,7 @@ else:
         return ctypes.WINFUNCTYPE(*args, **kwargs)
 
 
+# compat_open
 if sys.version_info < (3, 0):
     # open(file, mode='r', buffering=- 1, encoding=None, errors=None, newline=None, closefd=True) not: opener=None
     def compat_open(file_, *args, **kwargs):
@@ -3467,12 +3546,15 @@ except AttributeError:
     def compat_datetime_timedelta_total_seconds(td):
         return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6
 
+
 # optional decompression packages
+# compat_brotli
 # PyPi brotli package implements 'br' Content-Encoding
 try:
     import brotli as compat_brotli
 except ImportError:
     compat_brotli = None
+# compat_ncompress
 # PyPi ncompress package implements 'compress' Content-Encoding
 try:
     import ncompress as compat_ncompress
@@ -3495,6 +3577,7 @@ legacy = [
     'compat_getpass',
     'compat_parse_qs',
     'compat_realpath',
+    'compat_shlex_split',
     'compat_urllib_parse_parse_qs',
     'compat_urllib_parse_unquote',
     'compat_urllib_parse_unquote_plus',
@@ -3508,8 +3591,6 @@ legacy = [
 
 
 __all__ = [
-    'compat_html_parser_HTMLParseError',
-    'compat_html_parser_HTMLParser',
     'compat_Struct',
     'compat_base64_b64decode',
     'compat_basestring',
@@ -3518,13 +3599,9 @@ __all__ = [
     'compat_chr',
     'compat_collections_abc',
     'compat_collections_chain_map',
-    'compat_datetime_timedelta_total_seconds',
-    'compat_http_cookiejar',
-    'compat_http_cookiejar_Cookie',
-    'compat_http_cookies',
-    'compat_http_cookies_SimpleCookie',
     'compat_contextlib_suppress',
     'compat_ctypes_WINFUNCTYPE',
+    'compat_datetime_timedelta_total_seconds',
     'compat_etree_fromstring',
     'compat_etree_iterfind',
     'compat_filter',
@@ -3533,6 +3610,12 @@ __all__ = [
     'compat_getpass_getpass',
     'compat_html_entities',
     'compat_html_entities_html5',
+    'compat_html_parser_HTMLParseError',
+    'compat_html_parser_HTMLParser',
+    'compat_http_cookiejar',
+    'compat_http_cookiejar_Cookie',
+    'compat_http_cookies',
+    'compat_http_cookies_SimpleCookie',
     'compat_http_client',
     'compat_http_server',
     'compat_input',
@@ -3555,7 +3638,7 @@ __all__ = [
     'compat_register_utf8',
     'compat_setenv',
     'compat_shlex_quote',
-    'compat_shlex_split',
+    'compat_shutil_get_terminal_size',
     'compat_socket_create_connection',
     'compat_str',
     'compat_struct_pack',
@@ -3575,5 +3658,4 @@ __all__ = [
     'compat_xml_etree_register_namespace',
     'compat_xpath',
     'compat_zip',
-    'workaround_optparse_bug9161',
 ]

From 8738407d77f6da843f8f5ded1ccad73172b4abac Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 2 Mar 2025 13:36:05 +0000
Subject: [PATCH 816/860] [compat] Support zstd Content-Encoding * see RFC 8878
 7.2

---
 youtube_dl/compat.py | 8 ++++++++
 1 file changed, 8 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index e617dd511..6cd7abd24 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -3561,6 +3561,13 @@ try:
 except ImportError:
     compat_ncompress = None
 
+# compat_zstandard
+# PyPi zstandard package implements 'zstd' Content-Encoding (RFC 8878 7.2)
+try:
+    import zstandard as compat_zstandard
+except ImportError:
+    compat_zstandard = None
+
 
 legacy = [
     'compat_HTMLParseError',
@@ -3658,4 +3665,5 @@ __all__ = [
     'compat_xml_etree_register_namespace',
     'compat_xpath',
     'compat_zip',
+    'compat_zstandard',
 ]

From 974c7d7f349831cf32026ec57e75bc821843a07b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 7 Mar 2025 19:17:18 +0000
Subject: [PATCH 817/860] [compat] Fix inheriting from
 compat_collections_chain_map * see
 ytdl-org/youtube-dl#33079#issuecomment-2704038049

---
 youtube_dl/compat.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 6cd7abd24..8910a4dac 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -3473,11 +3473,12 @@ except ImportError:
         def new_child(self, m=None, **kwargs):
             m = m or {}
             m.update(kwargs)
-            return compat_collections_chain_map(m, *self.maps)
+            # support inheritance !
+            return type(self)(m, *self.maps)
 
         @property
         def parents(self):
-            return compat_collections_chain_map(*(self.maps[1:]))
+            return type(self)(*(self.maps[1:]))
 
 
 # compat_re_Pattern, compat_re_Match

From 94849bc997d232b344b0f3666198feec7b004b43 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 7 Mar 2025 19:32:56 +0000
Subject: [PATCH 818/860] [JSInterp] Improve Date processing * add JS_Date
 class implementing JS Date * support constructor args other than date string
 * support static methods of Date * Date objects are still automatically
 coerced to timestamp before using in JS.

---
 test/test_jsinterp.py  | 22 ++++++++++++
 youtube_dl/jsinterp.py | 76 +++++++++++++++++++++++++++++++++++++++++-
 2 files changed, 97 insertions(+), 1 deletion(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 6c34bc896..4c5256c4b 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -11,6 +11,7 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import math
 import re
+import time
 
 from youtube_dl.compat import compat_str as str
 from youtube_dl.jsinterp import JS_Undefined, JSInterpreter
@@ -208,6 +209,27 @@ class TestJSInterpreter(unittest.TestCase):
         self._test(jsi, 86000, args=['12/31/1969 18:01:26 MDT'])
         # epoch 0
         self._test(jsi, 0, args=['1 January 1970 00:00:00 UTC'])
+        # undefined
+        self._test(jsi, NaN, args=[JS_Undefined])
+        # y,m,d, ... - may fail with older dates lacking DST data
+        jsi = JSInterpreter('function f() { return new Date(%s); }'
+                            % ('2024, 5, 29, 2, 52, 12, 42',))
+        self._test(jsi, 1719625932042)
+        # no arg
+        self.assertAlmostEqual(JSInterpreter(
+            'function f() { return new Date() - 0; }').call_function('f'),
+            time.time() * 1000, delta=100)
+        # Date.now()
+        self.assertAlmostEqual(JSInterpreter(
+            'function f() { return Date.now(); }').call_function('f'),
+            time.time() * 1000, delta=100)
+        # Date.parse()
+        jsi = JSInterpreter('function f(dt) { return Date.parse(dt); }')
+        self._test(jsi, 0, args=['1 January 1970 00:00:00 UTC'])
+        # Date.UTC()
+        jsi = JSInterpreter('function f() { return Date.UTC(%s); }'
+                            % ('1970, 0, 1, 0, 0, 0, 0',))
+        self._test(jsi, 0)
 
     def test_call(self):
         jsi = JSInterpreter('''
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 2859bc734..c3ee3bb03 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -1,10 +1,12 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import calendar
 import itertools
 import json
 import operator
 import re
+import time
 
 from functools import update_wrapper, wraps
 
@@ -12,8 +14,10 @@ from .utils import (
     error_to_compat_str,
     ExtractorError,
     float_or_none,
+    int_or_none,
     js_to_json,
     remove_quotes,
+    str_or_none,
     unified_timestamp,
     variadic,
     write_string,
@@ -475,6 +479,73 @@ class JSInterpreter(object):
                 flags |= cls.RE_FLAGS[ch]
             return flags, expr[idx + 1:]
 
+    class JS_Date(object):
+        _t = None
+
+        @staticmethod
+        def __ymd_etc(*args, **kw_is_utc):
+            # args: year, monthIndex, day, hours, minutes, seconds, milliseconds
+            is_utc = kw_is_utc.get('is_utc', False)
+
+            args = list(args[:7])
+            args += [0] * (9 - len(args))
+            args[1] += 1  # month 0..11 -> 1..12
+            ms = args[6]
+            for i in range(6, 9):
+                args[i] = -1  # don't know
+            if is_utc:
+                args[-1] = 1
+            # TODO: [MDN] When a segment overflows or underflows its expected
+            # range, it usually "carries over to" or "borrows from" the higher segment.
+            try:
+                mktime = calendar.timegm if is_utc else time.mktime
+                return mktime(time.struct_time(args)) * 1000 + ms
+            except (OverflowError, ValueError):
+                return None
+
+        @classmethod
+        def UTC(cls, *args):
+            t = cls.__ymd_etc(*args, is_utc=True)
+            return _NaN if t is None else t
+
+        @staticmethod
+        def parse(date_str, **kw_is_raw):
+            is_raw = kw_is_raw.get('is_raw', False)
+
+            t = unified_timestamp(str_or_none(date_str), False)
+            return int(t * 1000) if t is not None else t if is_raw else _NaN
+
+        @staticmethod
+        def now(**kw_is_raw):
+            is_raw = kw_is_raw.get('is_raw', False)
+
+            t = time.time()
+            return int(t * 1000) if t is not None else t if is_raw else _NaN
+
+        def __init__(self, *args):
+            if not args:
+                args = [self.now(is_raw=True)]
+            if len(args) == 1:
+                if isinstance(args[0], JSInterpreter.JS_Date):
+                    self._t = int_or_none(args[0].valueOf(), default=None)
+                else:
+                    arg_type = _js_typeof(args[0])
+                    if arg_type == 'string':
+                        self._t = self.parse(args[0], is_raw=True)
+                    elif arg_type == 'number':
+                        self._t = int(args[0])
+            else:
+                self._t = self.__ymd_etc(*args)
+
+        def toString(self):
+            try:
+                return time.strftime('%a %b %0d %Y %H:%M:%S %Z%z', self._t).rstrip()
+            except TypeError:
+                return "Invalid Date"
+
+        def valueOf(self):
+            return _NaN if self._t is None else self._t
+
     @classmethod
     def __op_chars(cls):
         op_chars = set(';,[')
@@ -715,7 +786,7 @@ class JSInterpreter(object):
 
         new_kw, _, obj = expr.partition('new ')
         if not new_kw:
-            for klass, konstr in (('Date', lambda x: int(unified_timestamp(x, False) * 1000)),
+            for klass, konstr in (('Date', lambda *x: self.JS_Date(*x).valueOf()),
                                   ('RegExp', self.JS_RegExp),
                                   ('Error', self.Exception)):
                 if not obj.startswith(klass + '('):
@@ -1034,6 +1105,7 @@ class JSInterpreter(object):
                     'String': compat_str,
                     'Math': float,
                     'Array': list,
+                    'Date': self.JS_Date,
                 }
                 obj = local_vars.get(variable)
                 if obj in (JS_Undefined, None):
@@ -1086,6 +1158,8 @@ class JSInterpreter(object):
                         assertion(len(argvals) == 2, 'takes two arguments')
                         return argvals[0] ** argvals[1]
                     raise self.Exception('Unsupported Math method ' + member, expr=expr)
+                elif obj is self.JS_Date:
+                    return getattr(obj, member)(*argvals)
 
                 if member == 'split':
                     assertion(len(argvals) <= 2, 'takes at most two arguments')

From af049e309bfa47141a9788cd1730dd50dad6176d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 7 Mar 2025 19:37:56 +0000
Subject: [PATCH 819/860] [JSInterp] Handle undefined, etc, passed to JS_RegExp
 and Exception

---
 youtube_dl/jsinterp.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index c3ee3bb03..9b4157a43 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -408,6 +408,7 @@ class JSInterpreter(object):
     class Exception(ExtractorError):
         def __init__(self, msg, *args, **kwargs):
             expr = kwargs.pop('expr', None)
+            msg = str_or_none(msg, default='"None"')
             if expr is not None:
                 msg = '{0} in: {1!r:.100}'.format(msg.rstrip(), expr)
             super(JSInterpreter.Exception, self).__init__(msg, *args, **kwargs)
@@ -435,6 +436,7 @@ class JSInterpreter(object):
                 flags, _ = self.regex_flags(flags)
             # First, avoid https://github.com/python/cpython/issues/74534
             self.__self = None
+            pattern_txt = str_or_none(pattern_txt) or '(?:)'
             self.__pattern_txt = pattern_txt.replace('[[', r'[\[')
             self.__flags = flags
 

From 1dc27e1c3bda9cb8f44b805c89918aa7d11ffcdc Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 7 Mar 2025 19:40:53 +0000
Subject: [PATCH 820/860] [JSInterp] Make indexing error handling more
 conformant * by default TypeError -> undefined, else raise * set
 allow_undefined=True/False to override

---
 youtube_dl/jsinterp.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 9b4157a43..5a45fbb03 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -672,14 +672,15 @@ class JSInterpreter(object):
         except Exception as e:
             raise self.Exception('Failed to evaluate {left_val!r:.50} {op} {right_val!r:.50}'.format(**locals()), expr, cause=e)
 
-    def _index(self, obj, idx, allow_undefined=True):
+    def _index(self, obj, idx, allow_undefined=None):
         if idx == 'length' and isinstance(obj, list):
             return len(obj)
         try:
             return obj[int(idx)] if isinstance(obj, list) else obj[compat_str(idx)]
         except (TypeError, KeyError, IndexError) as e:
-            if allow_undefined:
-                # when is not allowed?
+            # allow_undefined is None gives correct behaviour
+            if allow_undefined or (
+                    allow_undefined is None and not isinstance(e, TypeError)):
                 return JS_Undefined
             raise self.Exception('Cannot get index {idx!r:.100}'.format(**locals()), expr=repr(obj), cause=e)
 

From 422b1b31cf398d60b4606fa57be8e39c1181932f Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 7 Mar 2025 20:00:58 +0000
Subject: [PATCH 821/860] [YouTube] Temporarily redirect from tce-style player
 JS

---
 youtube_dl/extractor/youtube.py | 9 ++++++++-
 1 file changed, 8 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5f8c08201..9e200105e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1607,9 +1607,16 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 webpage or '', 'player URL', fatal=False)
             if player_url:
                 ytcfgs = ytcfgs + ({'PLAYER_JS_URL': player_url},)
-        return traverse_obj(
+        player_url = traverse_obj(
             ytcfgs, (Ellipsis, 'PLAYER_JS_URL'), (Ellipsis, 'WEB_PLAYER_CONTEXT_CONFIGS', Ellipsis, 'jsUrl'),
             get_all=False, expected_type=lambda u: urljoin('https://www.youtube.com', u))
+        nplayer_url, is_tce = re.subn(r'(?<=/player_ias)_tce(?=\.vflset/)', '', player_url or '')
+        if is_tce:
+            # TODO: Add proper support for the 'tce' variant players
+            # See https://github.com/yt-dlp/yt-dlp/issues/12398
+            self.write_debug('Modifying tce player URL: {0}'.format(player_url))
+            return nplayer_url
+        return player_url
 
     def _download_player_url(self, video_id, fatal=False):
         res = self._download_webpage(

From 283dca56feb9f23978733810ab155472d6473c38 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 7 Mar 2025 21:02:28 +0000
Subject: [PATCH 822/860] [YouTube] Initially support tce-style player JS *
 resolves #33079

---
 test/test_youtube_signature.py  | 21 +++++++++++++++++----
 youtube_dl/extractor/youtube.py | 23 +++++++++++++----------
 2 files changed, 30 insertions(+), 14 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 67ef75fde..166614e62 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -223,6 +223,18 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/9c6dfc4a/player_ias.vflset/en_US/base.js',
         'jbu7ylIosQHyJyJV', 'uwI0ESiynAmhNg',
     ),
+    (
+        'https://www.youtube.com/s/player/f6e09c70/player_ias.vflset/en_US/base.js',
+        'W9HJZKktxuYoDTqW', 'jHbbkcaxm54',
+    ),
+    (
+        'https://www.youtube.com/s/player/f6e09c70/player_ias_tce.vflset/en_US/base.js',
+        'W9HJZKktxuYoDTqW', 'jHbbkcaxm54',
+    ),
+    (
+        'https://www.youtube.com/s/player/91201489/player_ias_tce.vflset/en_US/base.js',
+        'W9HJZKktxuYoDTqW', 'U48vOZHaeYS6vO',
+    ),
 ]
 
 
@@ -284,7 +296,7 @@ def t_factory(name, sig_func, url_pattern):
 
 
 def signature(jscode, sig_input):
-    func = YoutubeIE(FakeYDL())._parse_sig_js(jscode)
+    func = YoutubeIE(FakeYDL({'cachedir': False}))._parse_sig_js(jscode)
     src_sig = (
         compat_str(string.printable[:sig_input])
         if isinstance(sig_input, int) else sig_input)
@@ -292,9 +304,10 @@ def signature(jscode, sig_input):
 
 
 def n_sig(jscode, sig_input):
-    funcname = YoutubeIE(FakeYDL())._extract_n_function_name(jscode)
-    return JSInterpreter(jscode).call_function(
-        funcname, sig_input, _ytdl_do_not_return=sig_input)
+    ie = YoutubeIE(FakeYDL({'cachedir': False}))
+    jsi = JSInterpreter(jscode)
+    jsi, _, func_code = ie._extract_n_function_code_jsi(sig_input, jsi)
+    return ie._extract_n_function_from_code(jsi, func_code)(sig_input)
 
 
 make_sig_test = t_factory(
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 9e200105e..11bed6cae 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1607,16 +1607,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 webpage or '', 'player URL', fatal=False)
             if player_url:
                 ytcfgs = ytcfgs + ({'PLAYER_JS_URL': player_url},)
-        player_url = traverse_obj(
+        return traverse_obj(
             ytcfgs, (Ellipsis, 'PLAYER_JS_URL'), (Ellipsis, 'WEB_PLAYER_CONTEXT_CONFIGS', Ellipsis, 'jsUrl'),
             get_all=False, expected_type=lambda u: urljoin('https://www.youtube.com', u))
-        nplayer_url, is_tce = re.subn(r'(?<=/player_ias)_tce(?=\.vflset/)', '', player_url or '')
-        if is_tce:
-            # TODO: Add proper support for the 'tce' variant players
-            # See https://github.com/yt-dlp/yt-dlp/issues/12398
-            self.write_debug('Modifying tce player URL: {0}'.format(player_url))
-            return nplayer_url
-        return player_url
 
     def _download_player_url(self, video_id, fatal=False):
         res = self._download_webpage(
@@ -1858,12 +1851,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
         if func_code:
             return jsi, player_id, func_code
+        return self._extract_n_function_code_jsi(video_id, jsi, player_id)
 
-        func_name = self._extract_n_function_name(jscode)
+    def _extract_n_function_code_jsi(self, video_id, jsi, player_id=None):
+
+        var_ay = self._search_regex(
+            r'(?:[;\s]|^)\s*(var\s*[\w$]+\s*=\s*"[^"]+"\s*\.\s*split\("\{"\))(?=\s*[,;])',
+            jsi.code, 'useful values', default='')
+
+        func_name = self._extract_n_function_name(jsi.code)
 
         func_code = jsi.extract_function_code(func_name)
+        if var_ay:
+            func_code = (func_code[0], ';\n'.join((var_ay, func_code[1])))
 
-        self.cache.store('youtube-nsig', player_id, func_code)
+        if player_id:
+            self.cache.store('youtube-nsig', player_id, func_code)
         return jsi, player_id, func_code
 
     def _extract_n_function_from_code(self, jsi, func_code):

From 32f89de92b652bf246aa458a552c9bb397abef77 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 7 Mar 2025 21:03:54 +0000
Subject: [PATCH 823/860] [YouTube] Update TVHTML5 client parameters * resolves
 #33078

---
 youtube_dl/extractor/youtube.py | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 11bed6cae..6364e3aee 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -122,7 +122,8 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'INNERTUBE_CONTEXT': {
                 'client': {
                     'clientName': 'TVHTML5',
-                    'clientVersion': '7.20241201.18.00',
+                    'clientVersion': '7.20250120.19.00',
+                    'userAgent': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/Version',
                 },
             },
             'INNERTUBE_CONTEXT_CLIENT_NAME': 7,

From 420d53387cff54ea1fccca061438d59bdb50a39c Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 10 Mar 2025 11:44:06 +0000
Subject: [PATCH 824/860] [JSInterp] Improve tests * from yt-dlp/yt-dlp#12313 *
 also fix d7c2708

---
 test/test_jsinterp.py  | 21 ++++++++++++++++++---
 youtube_dl/jsinterp.py |  3 ++-
 2 files changed, 20 insertions(+), 4 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 4c5256c4b..3c9650ab6 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -212,9 +212,16 @@ class TestJSInterpreter(unittest.TestCase):
         # undefined
         self._test(jsi, NaN, args=[JS_Undefined])
         # y,m,d, ... - may fail with older dates lacking DST data
-        jsi = JSInterpreter('function f() { return new Date(%s); }'
-                            % ('2024, 5, 29, 2, 52, 12, 42',))
-        self._test(jsi, 1719625932042)
+        jsi = JSInterpreter(
+            'function f() { return new Date(%s); }'
+            % ('2024, 5, 29, 2, 52, 12, 42',))
+        self._test(jsi, (
+            1719625932042                           # UK value
+            + (
+                + 3600                              # back to GMT
+                + (time.altzone if time.daylight    # host's DST
+                   else time.timezone)
+            ) * 1000))
         # no arg
         self.assertAlmostEqual(JSInterpreter(
             'function f() { return new Date() - 0; }').call_function('f'),
@@ -485,6 +492,14 @@ class TestJSInterpreter(unittest.TestCase):
         self._test('function f(){return NaN << 42}', 0)
         self._test('function f(){return "21.9" << 1}', 42)
         self._test('function f(){return 21 << 4294967297}', 42)
+        self._test('function f(){return true << "5";}', 32)
+        self._test('function f(){return true << true;}', 2)
+        self._test('function f(){return "19" & "21.9";}', 17)
+        self._test('function f(){return "19" & false;}', 0)
+        self._test('function f(){return "11.0" >> "2.1";}', 2)
+        self._test('function f(){return 5 ^ 9;}', 12)
+        self._test('function f(){return 0.0 << NaN}', 0)
+        self._test('function f(){return null << undefined}', 0)
 
     def test_negative(self):
         self._test('function f(){return 2    *    -2.0    ;}', -4)
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 5a45fbb03..d9b33fa44 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -154,6 +154,7 @@ def _js_to_primitive(v):
     )
 
 
+# more exact: yt-dlp/yt-dlp#12110
 def _js_toString(v):
     return (
         'undefined' if v is JS_Undefined
@@ -162,7 +163,7 @@ def _js_toString(v):
         else 'null' if v is None
         # bool <= int: do this first
         else ('false', 'true')[v] if isinstance(v, bool)
-        else '{0:.7f}'.format(v).rstrip('.0') if isinstance(v, compat_numeric_types)
+        else re.sub(r'(?<=\d)\.?0*$', '', '{0:.7f}'.format(v)) if isinstance(v, compat_numeric_types)
         else _js_to_primitive(v))
 
 

From 37c2440d6a9a2a7226f6548ebc2b59160408e79a Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 21 Mar 2025 16:13:24 +0000
Subject: [PATCH 825/860] [YouTube] Update player client data thx seproDev
 (yt-dlp/yt-dlp#12603)

Co-authored-by: sepro <sepro@sepr0.com>
---
 youtube_dl/extractor/youtube.py | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 6364e3aee..e58b48abc 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -91,12 +91,12 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'INNERTUBE_CONTEXT': {
                 'client': {
                     'clientName': 'IOS',
-                    'clientVersion': '19.45.4',
+                    'clientVersion': '20.10.4',
                     'deviceMake': 'Apple',
                     'deviceModel': 'iPhone16,2',
-                    'userAgent': 'com.google.ios.youtube/19.45.4 (iPhone16,2; U; CPU iOS 18_1_0 like Mac OS X;)',
+                    'userAgent': 'com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)',
                     'osName': 'iPhone',
-                    'osVersion': '18.1.0.22B83',
+                    'osVersion': '18.3.2.22D82',
                 },
             },
             'INNERTUBE_CONTEXT_CLIENT_NAME': 5,
@@ -109,7 +109,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'INNERTUBE_CONTEXT': {
                 'client': {
                     'clientName': 'MWEB',
-                    'clientVersion': '2.20241202.07.00',
+                    'clientVersion': '2.20250311.03.00',
                     # mweb previously did not require PO Token with this UA
                     'userAgent': 'Mozilla/5.0 (iPad; CPU OS 16_7_10 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1,gzip(gfe)',
                 },
@@ -122,7 +122,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'INNERTUBE_CONTEXT': {
                 'client': {
                     'clientName': 'TVHTML5',
-                    'clientVersion': '7.20250120.19.00',
+                    'clientVersion': '7.20250312.16.00',
                     'userAgent': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/Version',
                 },
             },
@@ -133,7 +133,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             'INNERTUBE_CONTEXT': {
                 'client': {
                     'clientName': 'WEB',
-                    'clientVersion': '2.20241126.01.00',
+                    'clientVersion': '2.20250312.04.00',
                 },
             },
             'INNERTUBE_CONTEXT_CLIENT_NAME': 1,

From da7223d4aa42ff9fc680b0951d043dd03cec2d30 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 21 Mar 2025 16:26:25 +0000
Subject: [PATCH 826/860] [YouTube] Improve support for tce-style player JS *
 improve extraction of global "useful data" Array from player JS * also handle
 tv-player and add tests: thx seproDev (yt-dlp/yt-dlp#12684)

Co-Authored-By: sepro <sepro@sepr0.com>
---
 test/test_youtube_signature.py  | 28 ++++++++++++++++++++++++++--
 youtube_dl/extractor/youtube.py |  4 ++--
 2 files changed, 28 insertions(+), 4 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 166614e62..7726b1943 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -232,8 +232,32 @@ _NSIG_TESTS = [
         'W9HJZKktxuYoDTqW', 'jHbbkcaxm54',
     ),
     (
-        'https://www.youtube.com/s/player/91201489/player_ias_tce.vflset/en_US/base.js',
-        'W9HJZKktxuYoDTqW', 'U48vOZHaeYS6vO',
+        'https://www.youtube.com/s/player/643afba4/player_ias.vflset/en_US/base.js',
+        'W9HJZKktxuYoDTqW', 'larxUlagTRAcSw',
+    ),
+    (
+        'https://www.youtube.com/s/player/e7567ecf/player_ias_tce.vflset/en_US/base.js',
+        'Sy4aDGc0VpYRR9ew_', '5UPOT1VhoZxNLQ',
+    ),
+    (
+        'https://www.youtube.com/s/player/d50f54ef/player_ias_tce.vflset/en_US/base.js',
+        'Ha7507LzRmH3Utygtj', 'XFTb2HoeOE5MHg',
+    ),
+    (
+        'https://www.youtube.com/s/player/074a8365/player_ias_tce.vflset/en_US/base.js',
+        'Ha7507LzRmH3Utygtj', 'ufTsrE0IVYrkl8v',
+    ),
+    (
+        'https://www.youtube.com/s/player/643afba4/player_ias.vflset/en_US/base.js',
+        'N5uAlLqm0eg1GyHO', 'dCBQOejdq5s-ww',
+    ),
+    (
+        'https://www.youtube.com/s/player/69f581a5/tv-player-ias.vflset/tv-player-ias.js',
+        '-qIP447rVlTTwaZjY', 'KNcGOksBAvwqQg',
+    ),
+    (
+        'https://www.youtube.com/s/player/643afba4/tv-player-ias.vflset/tv-player-ias.js',
+        'ir9-V6cdbCiyKxhr', '2PL7ZDYAALMfmA',
     ),
 ]
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index e58b48abc..415354811 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -692,7 +692,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         'invidious': '|'.join(_INVIDIOUS_SITES),
     }
     _PLAYER_INFO_RE = (
-        r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/player',
+        r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})//(?:tv-)?player',
         r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
         r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
     )
@@ -1857,7 +1857,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
     def _extract_n_function_code_jsi(self, video_id, jsi, player_id=None):
 
         var_ay = self._search_regex(
-            r'(?:[;\s]|^)\s*(var\s*[\w$]+\s*=\s*"[^"]+"\s*\.\s*split\("\{"\))(?=\s*[,;])',
+            r'(?:[;\s]|^)\s*(var\s*[\w$]+\s*=\s*"(?:\\"|[^"])+"\s*\.\s*split\("\W+"\))(?=\s*[,;])',
             jsi.code, 'useful values', default='')
 
         func_name = self._extract_n_function_name(jsi.code)

From 2239ee796584ece265edef2df20782618a615056 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 25 Mar 2025 21:39:31 +0000
Subject: [PATCH 827/860] [YouTube] Get subtitles/automatic captions from both
 web and API responses

---
 youtube_dl/extractor/youtube.py | 16 ++++++++--------
 1 file changed, 8 insertions(+), 8 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 415354811..a3d3acc4d 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2136,7 +2136,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     video_details = merge_dicts(*traverse_obj(
                         (player_response, api_player_response),
                         (Ellipsis, 'videoDetails', T(dict))))
-                    player_response.update(api_player_response or {})
+                    player_response.update(filter_dict(
+                        api_player_response or {}, cndn=lambda k, _: k != 'captions'))
                     player_response['videoDetails'] = video_details
 
         def is_agegated(playability):
@@ -2566,8 +2567,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         }
 
         pctr = traverse_obj(
-            player_response,
-            ('captions', 'playerCaptionsTracklistRenderer', T(dict)))
+            (player_response, api_player_response),
+            (Ellipsis, 'captions', 'playerCaptionsTracklistRenderer', T(dict)))
         if pctr:
             def process_language(container, base_url, lang_code, query):
                 lang_subs = []
@@ -2584,20 +2585,19 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             def process_subtitles():
                 subtitles = {}
                 for caption_track in traverse_obj(pctr, (
-                        'captionTracks', lambda _, v: v.get('baseUrl'))):
+                        Ellipsis, 'captionTracks', lambda _, v: (
+                            v.get('baseUrl') and v.get('languageCode')))):
                     base_url = self._yt_urljoin(caption_track['baseUrl'])
                     if not base_url:
                         continue
+                    lang_code = caption_track['languageCode']
                     if caption_track.get('kind') != 'asr':
-                        lang_code = caption_track.get('languageCode')
-                        if not lang_code:
-                            continue
                         process_language(
                             subtitles, base_url, lang_code, {})
                         continue
                     automatic_captions = {}
                     for translation_language in traverse_obj(pctr, (
-                            'translationLanguages', lambda _, v: v.get('languageCode'))):
+                            Ellipsis, 'translationLanguages', lambda _, v: v.get('languageCode'))):
                         translation_language_code = translation_language['languageCode']
                         process_language(
                             automatic_captions, base_url, translation_language_code,

From 7dca08eff01c0e8a25be38ec76c63170f26ac118 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 25 Mar 2025 21:41:15 +0000
Subject: [PATCH 828/860] [YouTube] Also get original of translated automatic
 captions

---
 youtube_dl/extractor/youtube.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a3d3acc4d..50527c98c 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -2596,6 +2596,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                             subtitles, base_url, lang_code, {})
                         continue
                     automatic_captions = {}
+                    process_language(
+                        automatic_captions, base_url, lang_code, {})
                     for translation_language in traverse_obj(pctr, (
                             Ellipsis, 'translationLanguages', lambda _, v: v.get('languageCode'))):
                         translation_language_code = translation_language['languageCode']

From a464c159e61ab5c2c3babc8a4eb1d3e7923f1fe0 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 25 Mar 2025 21:49:21 +0000
Subject: [PATCH 829/860] [YouTube] Make `_extract_player_info()` use
 `_search_regex()`

---
 test/test_youtube_signature.py  |  3 ++-
 youtube_dl/extractor/youtube.py | 16 +++++++---------
 2 files changed, 9 insertions(+), 10 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 7726b1943..24d84dc20 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -280,8 +280,9 @@ class TestPlayerInfo(unittest.TestCase):
             ('https://s.ytimg.com/yts/jsbin/html5player-en_US-vflXGBaUN.js', 'vflXGBaUN'),
             ('https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js', 'vflKjOTVq'),
         )
+        ie = YoutubeIE(FakeYDL({'cachedir': False}))
         for player_url, expected_player_id in PLAYER_URLS:
-            player_id = YoutubeIE._extract_player_info(player_url)
+            player_id = ie._extract_player_info(player_url)
             self.assertEqual(player_id, expected_player_id)
 
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 50527c98c..a0adee7b8 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1626,15 +1626,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         """ Return a string representation of a signature """
         return '.'.join(compat_str(len(part)) for part in example_sig.split('.'))
 
-    @classmethod
-    def _extract_player_info(cls, player_url):
-        for player_re in cls._PLAYER_INFO_RE:
-            id_m = re.search(player_re, player_url)
-            if id_m:
-                break
-        else:
-            raise ExtractorError('Cannot identify player %r' % player_url)
-        return id_m.group('id')
+    def _extract_player_info(self, player_url):
+        try:
+            return self._search_regex(
+                self._PLAYER_INFO_RE, player_url, 'player info', group='id')
+        except ExtractorError as e:
+            raise ExtractorError(
+                'Cannot identify player %r' % (player_url,), cause=e)
 
     def _load_player(self, video_id, player_url, fatal=True, player_id=None):
         if not player_id:

From a4fc1151f125a2dfb89b2d1505838bcbb71aacc9 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 25 Mar 2025 21:56:32 +0000
Subject: [PATCH 830/860] [JSInterp] Improve indexing * catch invalid list
 index with `ValueError` (eg [1, 2]['ab'] -> undefined) * allow assignment
 outside existing list (eg var l = [1,2]; l[9] = 0;)

---
 youtube_dl/jsinterp.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index d9b33fa44..a6d4f8e75 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -678,7 +678,7 @@ class JSInterpreter(object):
             return len(obj)
         try:
             return obj[int(idx)] if isinstance(obj, list) else obj[compat_str(idx)]
-        except (TypeError, KeyError, IndexError) as e:
+        except (TypeError, KeyError, IndexError, ValueError) as e:
             # allow_undefined is None gives correct behaviour
             if allow_undefined or (
                     allow_undefined is None and not isinstance(e, TypeError)):
@@ -1038,6 +1038,10 @@ class JSInterpreter(object):
                     left_val = self._index(left_val, idx)
             if isinstance(idx, float):
                 idx = int(idx)
+            if isinstance(left_val, list) and len(left_val) <= int_or_none(idx, default=-1):
+                # JS Array is a sparsely assignable list
+                # TODO: handle extreme sparsity without memory bloat, eg using auxiliary dict
+                left_val.extend((idx - len(left_val) + 1) * [JS_Undefined])
             left_val[idx] = self._operator(
                 m.group('op'), self._index(left_val, idx) if m.group('op') else None,
                 m.group('expr'), expr, local_vars, allow_recursion)

From 087d8652302270764bf9679c7621db0d20ad5572 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 25 Mar 2025 22:14:08 +0000
Subject: [PATCH 831/860] [YouTube] Support new player URL patterns

---
 test/test_youtube_signature.py  | 12 ++++++++----
 youtube_dl/extractor/youtube.py |  6 +++---
 2 files changed, 11 insertions(+), 7 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 24d84dc20..385b718e0 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -302,8 +302,8 @@ class TestSignature(unittest.TestCase):
 def t_factory(name, sig_func, url_pattern):
     def make_tfunc(url, sig_input, expected_sig):
         m = url_pattern.match(url)
-        assert m, '%r should follow URL format' % url
-        test_id = m.group('id')
+        assert m, '{0!r} should follow URL format'.format(url)
+        test_id = re.sub(r'[/.-]', '_', m.group('id') or m.group('compat_id'))
 
         def test_func(self):
             basename = 'player-{0}-{1}.js'.format(name, test_id)
@@ -336,12 +336,16 @@ def n_sig(jscode, sig_input):
 
 
 make_sig_test = t_factory(
-    'signature', signature, re.compile(r'.*(?:-|/player/)(?P<id>[a-zA-Z0-9_-]+)(?:/.+\.js|(?:/watch_as3|/html5player)?\.[a-z]+)$'))
+    'signature', signature,
+    re.compile(r'''(?x)
+        .+/(?P<h5>html5)?player(?(h5)(?:-en_US)?-|/)(?P<id>[a-zA-Z0-9/._-]+)
+        (?(h5)/(?:watch_as3|html5player))?\.js$
+    '''))
 for test_spec in _SIG_TESTS:
     make_sig_test(*test_spec)
 
 make_nsig_test = t_factory(
-    'nsig', n_sig, re.compile(r'.+/player/(?P<id>[a-zA-Z0-9_-]+)/.+.js$'))
+    'nsig', n_sig, re.compile(r'.+/player/(?P<id>[a-zA-Z0-9_/.-]+)\.js$'))
 for test_spec in _NSIG_TESTS:
     make_nsig_test(*test_spec)
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a0adee7b8..4347663ad 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -692,9 +692,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         'invidious': '|'.join(_INVIDIOUS_SITES),
     }
     _PLAYER_INFO_RE = (
-        r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})//(?:tv-)?player',
-        r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
-        r'\b(?P<id>vfl[a-zA-Z0-9_-]+)\b.*?\.js$',
+        r'/s/player/(?P<id>[a-zA-Z0-9_-]{8,})/(?:tv-)?player',
+        r'/(?P<id>[a-zA-Z0-9_-]{8,})/player(?:_ias(?:_tce)?\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$',
+        r'\b(?P<id>vfl[a-zA-Z0-9_-]{6,})\b.*?\.js$',
     )
     _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt')
 

From c55dbf48384f9721ae8eb07fd914ccf2d3c823b0 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 25 Mar 2025 22:16:19 +0000
Subject: [PATCH 832/860] [YouTube] Update signature extraction for players
 `643afba4`, `363db69b`

---
 youtube_dl/extractor/youtube.py | 28 +++++++++++++++++++---------
 youtube_dl/jsinterp.py          | 10 ++++++----
 2 files changed, 25 insertions(+), 13 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 4347663ad..5addfe7c6 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1709,6 +1709,18 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                 '    return %s\n') % (signature_id_tuple, expr_code)
         self.to_screen('Extracted signature function:\n' + code)
 
+    def _extract_sig_fn(self, jsi, funcname):
+        var_ay = self._search_regex(
+            r'''(?:\*/|\{|\n|^)\s*(?:'[^']+'\s*;\s*)(var\s*[\w$]+\s*=\s*('|")(?:\\\2|(?!\2).)+\2\s*\.\s*split\(('|")\W+\3\))(?=\s*[,;])''',
+            jsi.code, 'useful values', default='')
+
+        sig_fn = jsi.extract_function_code(funcname)
+
+        if var_ay:
+            sig_fn = (sig_fn[0], ';\n'.join((var_ay, sig_fn[1])))
+
+        return sig_fn
+
     def _parse_sig_js(self, jscode):
         # Examples where `sig` is funcname:
         # sig=function(a){a=a.split(""); ... ;return a.join("")};
@@ -1734,8 +1746,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             jscode, 'Initial JS player signature function name', group='sig')
 
         jsi = JSInterpreter(jscode)
-        initial_function = jsi.extract_function(funcname)
-        return lambda s: initial_function([s])
+
+        initial_function = self._extract_sig_fn(jsi, funcname)
+
+        func = jsi.extract_function_from_code(*initial_function)
+
+        return lambda s: func([s])
 
     def _cached(self, func, *cache_id):
         def inner(*args, **kwargs):
@@ -1854,15 +1870,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     def _extract_n_function_code_jsi(self, video_id, jsi, player_id=None):
 
-        var_ay = self._search_regex(
-            r'(?:[;\s]|^)\s*(var\s*[\w$]+\s*=\s*"(?:\\"|[^"])+"\s*\.\s*split\("\W+"\))(?=\s*[,;])',
-            jsi.code, 'useful values', default='')
-
         func_name = self._extract_n_function_name(jsi.code)
 
-        func_code = jsi.extract_function_code(func_name)
-        if var_ay:
-            func_code = (func_code[0], ';\n'.join((var_ay, func_code[1])))
+        func_code = self._extract_sig_fn(jsi, func_name)
 
         if player_id:
             self.cache.store('youtube-nsig', player_id, func_code)
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index a6d4f8e75..186d2594a 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -1368,19 +1368,21 @@ class JSInterpreter(object):
         code, _ = self._separate_at_paren(func_m.group('code'))  # refine the match
         return self.build_arglist(func_m.group('args')), code
 
-    def extract_function(self, funcname):
+    def extract_function(self, funcname, *global_stack):
         return function_with_repr(
-            self.extract_function_from_code(*self.extract_function_code(funcname)),
+            self.extract_function_from_code(*itertools.chain(
+                self.extract_function_code(funcname), global_stack)),
             'F<%s>' % (funcname,))
 
     def extract_function_from_code(self, argnames, code, *global_stack):
         local_vars = {}
 
+        start = None
         while True:
-            mobj = re.search(r'function\((?P<args>[^)]*)\)\s*{', code)
+            mobj = re.search(r'function\((?P<args>[^)]*)\)\s*{', code[start:])
             if mobj is None:
                 break
-            start, body_start = mobj.span()
+            start, body_start = ((start or 0) + x for x in mobj.span())
             body, remaining = self._separate_at_paren(code[body_start - 1:])
             name = self._named_object(local_vars, self.extract_function_from_code(
                 [x.strip() for x in mobj.group('args').split(',')],

From b982d77d0b7bd21ead6d8b3210fd656881197453 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 25 Mar 2025 22:18:47 +0000
Subject: [PATCH 833/860] [YouTube] Align signature tests with yt-dlp thx
 bashonly, yt-dlp/yt-dlp#12725

---
 test/test_youtube_signature.py | 28 ++++++++++++++++++++++------
 1 file changed, 22 insertions(+), 6 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 385b718e0..696f832e4 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -84,6 +84,16 @@ _SIG_TESTS = [
         '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
         '0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xxAj7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJ2OySqa0q',
     ),
+    (
+        'https://www.youtube.com/s/player/643afba4/tv-player-ias.vflset/tv-player-ias.js',
+        '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+        'AAOAOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7vgpDL0QwbdV06sCIEzpWqMGkFR20CFOS21Tp-7vj_EMu-m37KtXJoOy1',
+    ),
+    (
+        'https://www.youtube.com/s/player/363db69b/player_ias.vflset/en_US/base.js',
+        '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+        '0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpz2ICs6EVdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+    ),
 ]
 
 _NSIG_TESTS = [
@@ -153,7 +163,7 @@ _NSIG_TESTS = [
     ),
     (
         'https://www.youtube.com/s/player/c57c113c/player_ias.vflset/en_US/base.js',
-        '-Txvy6bT5R6LqgnQNx', 'dcklJCnRUHbgSg',
+        'M92UUMHa8PdvPd3wyM', '3hPqLJsiNZx7yA',
     ),
     (
         'https://www.youtube.com/s/player/5a3b6271/player_ias.vflset/en_US/base.js',
@@ -173,7 +183,7 @@ _NSIG_TESTS = [
     ),
     (
         'https://www.youtube.com/s/player/cfa9e7cb/player_ias.vflset/en_US/base.js',
-        'qO0NiMtYQ7TeJnfFG2', 'k9cuJDHNS5O7kQ',
+        'aCi3iElgd2kq0bxVbQ', 'QX1y8jGb2IbZ0w',
     ),
     (
         'https://www.youtube.com/s/player/8c7583ff/player_ias.vflset/en_US/base.js',
@@ -231,10 +241,6 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/f6e09c70/player_ias_tce.vflset/en_US/base.js',
         'W9HJZKktxuYoDTqW', 'jHbbkcaxm54',
     ),
-    (
-        'https://www.youtube.com/s/player/643afba4/player_ias.vflset/en_US/base.js',
-        'W9HJZKktxuYoDTqW', 'larxUlagTRAcSw',
-    ),
     (
         'https://www.youtube.com/s/player/e7567ecf/player_ias_tce.vflset/en_US/base.js',
         'Sy4aDGc0VpYRR9ew_', '5UPOT1VhoZxNLQ',
@@ -259,6 +265,14 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/643afba4/tv-player-ias.vflset/tv-player-ias.js',
         'ir9-V6cdbCiyKxhr', '2PL7ZDYAALMfmA',
     ),
+    (
+        'https://www.youtube.com/s/player/643afba4/player_ias.vflset/en_US/base.js',
+        'W9HJZKktxuYoDTqW', 'larxUlagTRAcSw',
+    ),
+    (
+        'https://www.youtube.com/s/player/363db69b/player_ias.vflset/en_US/base.js',
+        'eWYu5d5YeY_4LyEDc', 'XJQqf-N7Xra3gg',
+    ),
 ]
 
 
@@ -271,6 +285,8 @@ class TestPlayerInfo(unittest.TestCase):
             ('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-phone-en_US.vflset/base.js', '64dddad9'),
             ('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-phone-de_DE.vflset/base.js', '64dddad9'),
             ('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-tablet-en_US.vflset/base.js', '64dddad9'),
+            ('https://www.youtube.com/s/player/e7567ecf/player_ias_tce.vflset/en_US/base.js', 'e7567ecf'),
+            ('https://www.youtube.com/s/player/643afba4/tv-player-ias.vflset/tv-player-ias.js', '643afba4'),
             # obsolete
             ('https://www.youtube.com/yts/jsbin/player_ias-vfle4-e03/en_US/base.js', 'vfle4-e03'),
             ('https://www.youtube.com/yts/jsbin/player_ias-vfl49f_g4/en_US/base.js', 'vfl49f_g4'),

From 1bc45b8b6c0c853e0f3d74346551fec0708c733d Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Tue, 25 Mar 2025 22:30:08 +0000
Subject: [PATCH 834/860] [JSInterp] Use `,` for join() with null/undefined
 argument Eg: [1,2,3].join(null) -> '1,2,3'

---
 youtube_dl/jsinterp.py | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 186d2594a..f0c4fa928 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -1208,9 +1208,10 @@ class JSInterpreter(object):
                 elif member == 'join':
                     assertion(isinstance(obj, list), 'must be applied on a list')
                     assertion(len(argvals) <= 1, 'takes at most one argument')
-                    return (',' if len(argvals) == 0 else argvals[0]).join(
-                        ('' if x in (None, JS_Undefined) else _js_toString(x))
-                        for x in obj)
+                    return (',' if len(argvals) == 0 or argvals[0] in (None, JS_Undefined)
+                            else argvals[0]).join(
+                                ('' if x in (None, JS_Undefined) else _js_toString(x))
+                                for x in obj)
                 elif member == 'reverse':
                     assertion(not argvals, 'does not take any arguments')
                     obj.reverse()

From 2b4fbfce25902d557b86b003cf48f738129efce4 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 26 Mar 2025 02:27:25 +0000
Subject: [PATCH 835/860] [YouTube] Support player `4fcd6e4a` thx seproDev,
 bashonly: yt-dlp/yt-dlp#12748

---
 test/test_youtube_signature.py  | 13 +++++++++++++
 youtube_dl/extractor/youtube.py |  9 +++++++--
 2 files changed, 20 insertions(+), 2 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 696f832e4..8fdcce8d4 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -94,6 +94,11 @@ _SIG_TESTS = [
         '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
         '0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpz2ICs6EVdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
     ),
+    (
+        'https://www.youtube.com/s/player/4fcd6e4a/player_ias.vflset/en_US/base.js',
+        '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+        'wAOAOq0QJ8ARAIgXmPlOPSBkkUs1bYFYlJCfe29xx8q7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0',
+    ),
 ]
 
 _NSIG_TESTS = [
@@ -273,6 +278,14 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/363db69b/player_ias.vflset/en_US/base.js',
         'eWYu5d5YeY_4LyEDc', 'XJQqf-N7Xra3gg',
     ),
+    (
+        'https://www.youtube.com/s/player/4fcd6e4a/player_ias.vflset/en_US/base.js',
+        'o_L251jm8yhZkWtBW', 'lXoxI3XvToqn6A',
+    ),
+    (
+        'https://www.youtube.com/s/player/4fcd6e4a/tv-player-ias.vflset/tv-player-ias.js',
+        'o_L251jm8yhZkWtBW', 'lXoxI3XvToqn6A',
+    ),
 ]
 
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 5addfe7c6..ce97fd75b 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1711,8 +1711,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     def _extract_sig_fn(self, jsi, funcname):
         var_ay = self._search_regex(
-            r'''(?:\*/|\{|\n|^)\s*(?:'[^']+'\s*;\s*)(var\s*[\w$]+\s*=\s*('|")(?:\\\2|(?!\2).)+\2\s*\.\s*split\(('|")\W+\3\))(?=\s*[,;])''',
-            jsi.code, 'useful values', default='')
+            r'''(?x)
+                (?:\*/|\{|\n|^)\s*(?:'[^']+'\s*;\s*)
+                    (var\s*[\w$]+\s*=\s*(?:
+                        ('|")(?:\\\2|(?!\2).)+\2\s*\.\s*split\(\s*('|")\W+\3\s*\)|
+                        \[\s*(?:('|")(?:\\\4|(?!\4).)*\4\s*(?:(?=\])|,\s*))+\]
+                    ))(?=\s*[,;])
+            ''', jsi.code, 'useful values', default='')
 
         sig_fn = jsi.extract_function_code(funcname)
 

From c1ea7f5a242dbaad3a4646d73bb1cae22439a788 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 26 Mar 2025 12:17:49 +0000
Subject: [PATCH 836/860] [ITV] Mark ITVX not working * update old shim *
 correct [_]IE_DESC

---
 youtube_dl/extractor/itv.py | 17 +++++------------
 1 file changed, 5 insertions(+), 12 deletions(-)

diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py
index c64af3be6..2510ad887 100644
--- a/youtube_dl/extractor/itv.py
+++ b/youtube_dl/extractor/itv.py
@@ -35,15 +35,6 @@ from ..utils import (
 
 class ITVBaseIE(InfoExtractor):
 
-    def _search_nextjs_data(self, webpage, video_id, **kw):
-        transform_source = kw.pop('transform_source', None)
-        fatal = kw.pop('fatal', True)
-        return self._parse_json(
-            self._search_regex(
-                r'''<script\b[^>]+\bid=('|")__NEXT_DATA__\1[^>]*>(?P<js>[^<]+)</script>''',
-                webpage, 'next.js data', group='js', fatal=fatal, **kw),
-            video_id, transform_source=transform_source, fatal=fatal)
-
     def __handle_request_webpage_error(self, err, video_id=None, errnote=None, fatal=True):
         if errnote is False:
             return False
@@ -109,7 +100,9 @@ class ITVBaseIE(InfoExtractor):
 
 class ITVIE(ITVBaseIE):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/(?:(?P<w>watch)|hub)/[^/]+/(?(w)[\w-]+/)(?P<id>\w+)'
-    _IE_DESC = 'ITVX'
+    IE_DESC = 'ITVX'
+    _WORKING = False
+
     _TESTS = [{
         'note': 'Hub URLs redirect to ITVX',
         'url': 'https://www.itv.com/hub/liar/2a4547a0012',
@@ -270,7 +263,7 @@ class ITVIE(ITVBaseIE):
                 'ext': determine_ext(href, 'vtt'),
             })
 
-        next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default='{}')
+        next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default={})
         video_data.update(traverse_obj(next_data, ('props', 'pageProps', ('title', 'episode')), expected_type=dict)[0] or {})
         title = traverse_obj(video_data, 'headerTitle', 'episodeTitle')
         info = self._og_extract(webpage, require_title=not title)
@@ -323,7 +316,7 @@ class ITVIE(ITVBaseIE):
 
 class ITVBTCCIE(ITVBaseIE):
     _VALID_URL = r'https?://(?:www\.)?itv\.com/(?!(?:watch|hub)/)(?:[^/]+/)+(?P<id>[^/?#&]+)'
-    _IE_DESC = 'ITV articles: News, British Touring Car Championship'
+    IE_DESC = 'ITV articles: News, British Touring Car Championship'
     _TESTS = [{
         'note': 'British Touring Car Championship',
         'url': 'https://www.itv.com/btcc/articles/btcc-2018-all-the-action-from-brands-hatch',

From 4e714f9df1ed2cccd51df60d45ff5504abe827b7 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Wed, 26 Mar 2025 12:47:19 +0000
Subject: [PATCH 837/860] [Misc] Correct [_]IE_DESC/NAME in a few IEs * thx
 seproDev, yt-dlp/yt-dlp/pull/12694/commits/ae69e3c * also add documenting
 comment in `InfoExtractor`

---
 youtube_dl/extractor/bokecc.py     | 2 +-
 youtube_dl/extractor/cloudy.py     | 2 +-
 youtube_dl/extractor/common.py     | 2 ++
 youtube_dl/extractor/senateisvp.py | 2 +-
 4 files changed, 5 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/extractor/bokecc.py b/youtube_dl/extractor/bokecc.py
index 6017e8344..4b8bef391 100644
--- a/youtube_dl/extractor/bokecc.py
+++ b/youtube_dl/extractor/bokecc.py
@@ -32,7 +32,7 @@ class BokeCCBaseIE(InfoExtractor):
 
 
 class BokeCCIE(BokeCCBaseIE):
-    _IE_DESC = 'CC视频'
+    IE_DESC = 'CC视频'
     _VALID_URL = r'https?://union\.bokecc\.com/playvideo\.bo\?(?P<query>.*)'
 
     _TESTS = [{
diff --git a/youtube_dl/extractor/cloudy.py b/youtube_dl/extractor/cloudy.py
index 85ca20ecc..d39a9a5c2 100644
--- a/youtube_dl/extractor/cloudy.py
+++ b/youtube_dl/extractor/cloudy.py
@@ -9,7 +9,7 @@ from ..utils import (
 
 
 class CloudyIE(InfoExtractor):
-    _IE_DESC = 'cloudy.ec'
+    IE_DESC = 'cloudy.ec'
     _VALID_URL = r'https?://(?:www\.)?cloudy\.ec/(?:v/|embed\.php\?.*?\bid=)(?P<id>[A-Za-z0-9]+)'
     _TESTS = [{
         'url': 'https://www.cloudy.ec/v/af511e2527aac',
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 78704b557..cb67b976d 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -422,6 +422,8 @@ class InfoExtractor(object):
     _GEO_COUNTRIES = None
     _GEO_IP_BLOCKS = None
     _WORKING = True
+    # supply this in public subclasses: used in supported sites list, etc
+    # IE_DESC = 'short description of IE'
 
     def __init__(self, downloader=None):
         """Constructor. Receives an optional downloader."""
diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py
index db5ef8b57..b8ac58713 100644
--- a/youtube_dl/extractor/senateisvp.py
+++ b/youtube_dl/extractor/senateisvp.py
@@ -47,7 +47,7 @@ class SenateISVPIE(InfoExtractor):
         ['vetaff', '76462', 'http://vetaff-f.akamaihd.net'],
         ['arch', '', 'http://ussenate-f.akamaihd.net/']
     ]
-    _IE_NAME = 'senate.gov'
+    IE_NAME = 'senate.gov'
     _VALID_URL = r'https?://(?:www\.)?senate\.gov/isvp/?\?(?P<qs>.+)'
     _TESTS = [{
         'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png',

From 3eb8d22ddb8982ca4fb56bb7a8d6517538bf14c6 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 31 Mar 2025 04:21:09 +0100
Subject: [PATCH 838/860] [JSInterp] Temporary fix for #33102

---
 youtube_dl/jsinterp.py | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index f0c4fa928..69c8f77ca 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -686,6 +686,8 @@ class JSInterpreter(object):
             raise self.Exception('Cannot get index {idx!r:.100}'.format(**locals()), expr=repr(obj), cause=e)
 
     def _dump(self, obj, namespace):
+        if obj is JS_Undefined:
+            return 'undefined'
         try:
             return json.dumps(obj)
         except TypeError:

From 9a6ddece4dca4d5b7a8374a82f72d778750d6b88 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Apr 2025 04:04:01 +0100
Subject: [PATCH 839/860] [core] Refactor message routines to align better with
 yt-dlp * in particular, support `only_once` in the same methods

---
 youtube_dl/YoutubeDL.py        | 37 ++++++++++++++++++++--------------
 youtube_dl/extractor/common.py | 14 ++-----------
 2 files changed, 24 insertions(+), 27 deletions(-)

diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py
index 9e5620eef..8367b6e53 100755
--- a/youtube_dl/YoutubeDL.py
+++ b/youtube_dl/YoutubeDL.py
@@ -540,10 +540,14 @@ class YoutubeDL(object):
         """Print message to stdout if not in quiet mode."""
         return self.to_stdout(message, skip_eol, check_quiet=True)
 
-    def _write_string(self, s, out=None):
+    def _write_string(self, s, out=None, only_once=False, _cache=set()):
+        if only_once and s in _cache:
+            return
         write_string(s, out=out, encoding=self.params.get('encoding'))
+        if only_once:
+            _cache.add(s)
 
-    def to_stdout(self, message, skip_eol=False, check_quiet=False):
+    def to_stdout(self, message, skip_eol=False, check_quiet=False, only_once=False):
         """Print message to stdout if not in quiet mode."""
         if self.params.get('logger'):
             self.params['logger'].debug(message)
@@ -552,9 +556,9 @@ class YoutubeDL(object):
             terminator = ['\n', ''][skip_eol]
             output = message + terminator
 
-            self._write_string(output, self._screen_file)
+            self._write_string(output, self._screen_file, only_once=only_once)
 
-    def to_stderr(self, message):
+    def to_stderr(self, message, only_once=False):
         """Print message to stderr."""
         assert isinstance(message, compat_str)
         if self.params.get('logger'):
@@ -562,7 +566,7 @@ class YoutubeDL(object):
         else:
             message = self._bidi_workaround(message)
             output = message + '\n'
-            self._write_string(output, self._err_file)
+            self._write_string(output, self._err_file, only_once=only_once)
 
     def to_console_title(self, message):
         if not self.params.get('consoletitle', False):
@@ -641,18 +645,11 @@ class YoutubeDL(object):
             raise DownloadError(message, exc_info)
         self._download_retcode = 1
 
-    def report_warning(self, message, only_once=False, _cache={}):
+    def report_warning(self, message, only_once=False):
         '''
         Print the message to stderr, it will be prefixed with 'WARNING:'
         If stderr is a tty file the 'WARNING:' will be colored
         '''
-        if only_once:
-            m_hash = hash((self, message))
-            m_cnt = _cache.setdefault(m_hash, 0)
-            _cache[m_hash] = m_cnt + 1
-            if m_cnt > 0:
-                return
-
         if self.params.get('logger') is not None:
             self.params['logger'].warning(message)
         else:
@@ -663,7 +660,7 @@ class YoutubeDL(object):
             else:
                 _msg_header = 'WARNING:'
             warning_message = '%s %s' % (_msg_header, message)
-            self.to_stderr(warning_message)
+            self.to_stderr(warning_message, only_once=only_once)
 
     def report_error(self, message, *args, **kwargs):
         '''
@@ -677,6 +674,16 @@ class YoutubeDL(object):
         kwargs['message'] = '%s %s' % (_msg_header, message)
         self.trouble(*args, **kwargs)
 
+    def write_debug(self, message, only_once=False):
+        '''Log debug message or Print message to stderr'''
+        if not self.params.get('verbose', False):
+            return
+        message = '[debug] {0}'.format(message)
+        if self.params.get('logger'):
+            self.params['logger'].debug(message)
+        else:
+            self.to_stderr(message, only_once)
+
     def report_unscoped_cookies(self, *args, **kwargs):
         # message=None, tb=False, is_error=False
         if len(args) <= 2:
@@ -2514,7 +2521,7 @@ class YoutubeDL(object):
                 self.get_encoding()))
         write_string(encoding_str, encoding=None)
 
-        writeln_debug = lambda *s: self._write_string('[debug] %s\n' % (''.join(s), ))
+        writeln_debug = lambda *s: self.write_debug(''.join(s))
         writeln_debug('youtube-dl version ', __version__)
         if _LAZY_LOADER:
             writeln_debug('Lazy loading extractors enabled')
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index cb67b976d..cbc4e968a 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -976,19 +976,9 @@ class InfoExtractor(object):
         """Print msg to screen, prefixing it with '[ie_name]'"""
         self._downloader.to_screen(self.__ie_msg(msg))
 
-    def write_debug(self, msg, only_once=False, _cache=[]):
+    def write_debug(self, msg, only_once=False):
         '''Log debug message or Print message to stderr'''
-        if not self.get_param('verbose', False):
-            return
-        message = '[debug] ' + self.__ie_msg(msg)
-        logger = self.get_param('logger')
-        if logger:
-            logger.debug(message)
-        else:
-            if only_once and hash(message) in _cache:
-                return
-            self._downloader.to_stderr(message)
-            _cache.append(hash(message))
+        self._downloader.write_debug(self.__ie_msg(msg), only_once=only_once)
 
     # name, default=None, *args, **kwargs
     def get_param(self, name, *args, **kwargs):

From add46228701d78444e20afdde4beafa3268344f2 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Apr 2025 04:09:56 +0100
Subject: [PATCH 840/860] [compat] Add compat_os_makedirs * support exists_ok
 parameter in Py < 3.2

---
 youtube_dl/compat.py | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 8910a4dac..72b2f8f78 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -3120,6 +3120,21 @@ else:
 compat_os_path_expanduser = compat_expanduser
 
 
+# compat_os_makedirs
+try:
+    os.makedirs('.', exist_ok=True)
+    compat_os_makedirs = os.makedirs
+except TypeError:  # < Py3.2
+    from errno import EEXIST as _errno_EEXIST
+
+    def compat_os_makedirs(name, mode=0o777, exist_ok=False):
+        try:
+            return os.makedirs(name, mode=mode)
+        except OSError as ose:
+            if not (exist_ok and ose.errno == _errno_EEXIST):
+                raise
+
+
 # compat_os_path_realpath
 if compat_os_name == 'nt' and sys.version_info < (3, 8):
     # os.path.realpath on Windows does not follow symbolic links
@@ -3637,6 +3652,7 @@ __all__ = [
     'compat_numeric_types',
     'compat_open',
     'compat_ord',
+    'compat_os_makedirs',
     'compat_os_name',
     'compat_os_path_expanduser',
     'compat_os_path_realpath',

From cedeeed56f6ed61646a33e06397f9a059dbc9d90 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Apr 2025 04:42:14 +0100
Subject: [PATCH 841/860] [cache] Align further with yt-dlp * use
 compat_os_makedirs * support non-ASCII characters in cache key * improve
 logging

---
 test/test_cache.py  |  2 +-
 youtube_dl/cache.py | 70 ++++++++++++++++++++++++++-------------------
 2 files changed, 41 insertions(+), 31 deletions(-)

diff --git a/test/test_cache.py b/test/test_cache.py
index 931074aa1..21dfb6cfc 100644
--- a/test/test_cache.py
+++ b/test/test_cache.py
@@ -63,7 +63,7 @@ class TestCache(unittest.TestCase):
         obj = {'x': 1, 'y': ['ä', '\\a', True]}
         c.store('test_cache', 'k.', obj)
         self.assertEqual(c.load('test_cache', 'k.', min_ver='1970.01.01'), obj)
-        new_version = '.'.join(('%d' % ((v + 1) if i == 0 else v, )) for i, v in enumerate(version_tuple(__version__)))
+        new_version = '.'.join(('%0.2d' % ((v + 1) if i == 0 else v, )) for i, v in enumerate(version_tuple(__version__)))
         self.assertIs(c.load('test_cache', 'k.', min_ver=new_version), None)
 
 
diff --git a/youtube_dl/cache.py b/youtube_dl/cache.py
index 54123da0e..6169461b9 100644
--- a/youtube_dl/cache.py
+++ b/youtube_dl/cache.py
@@ -1,6 +1,6 @@
+# coding: utf-8
 from __future__ import unicode_literals
 
-import errno
 import json
 import os
 import re
@@ -8,14 +8,17 @@ import shutil
 import traceback
 
 from .compat import (
+    compat_contextlib_suppress,
     compat_getenv,
     compat_open as open,
+    compat_os_makedirs,
+    compat_urllib_parse,
 )
 from .utils import (
     error_to_compat_str,
     expand_path,
     is_outdated_version,
-    try_get,
+    traverse_obj,
     write_json_file,
 )
 from .version import __version__
@@ -30,23 +33,36 @@ class Cache(object):
     def __init__(self, ydl):
         self._ydl = ydl
 
+    def _write_debug(self, *args, **kwargs):
+        self._ydl.write_debug(*args, **kwargs)
+
+    def _report_warning(self, *args, **kwargs):
+        self._ydl.report_warning(*args, **kwargs)
+
+    def _to_screen(self, *args, **kwargs):
+        self._ydl.to_screen(*args, **kwargs)
+
+    def _get_params(self, k, default=None):
+        return self._ydl.params.get(k, default)
+
     def _get_root_dir(self):
-        res = self._ydl.params.get('cachedir')
+        res = self._get_params('cachedir')
         if res is None:
             cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
             res = os.path.join(cache_root, self._YTDL_DIR)
         return expand_path(res)
 
     def _get_cache_fn(self, section, key, dtype):
-        assert re.match(r'^[a-zA-Z0-9_.-]+$', section), \
+        assert re.match(r'^[\w.-]+$', section), \
             'invalid section %r' % section
-        assert re.match(r'^[a-zA-Z0-9_.-]+$', key), 'invalid key %r' % key
+        key = compat_urllib_parse.quote(
+            key, safe='').replace('%', ',')  # encode non-ascii characters
         return os.path.join(
             self._get_root_dir(), section, '%s.%s' % (key, dtype))
 
     @property
     def enabled(self):
-        return self._ydl.params.get('cachedir') is not False
+        return self._get_params('cachedir') is not False
 
     def store(self, section, key, data, dtype='json'):
         assert dtype in ('json',)
@@ -56,61 +72,55 @@ class Cache(object):
 
         fn = self._get_cache_fn(section, key, dtype)
         try:
-            try:
-                os.makedirs(os.path.dirname(fn))
-            except OSError as ose:
-                if ose.errno != errno.EEXIST:
-                    raise
+            compat_os_makedirs(os.path.dirname(fn), exist_ok=True)
+            self._write_debug('Saving {section}.{key} to cache'.format(section=section, key=key))
             write_json_file({self._VERSION_KEY: __version__, 'data': data}, fn)
         except Exception:
             tb = traceback.format_exc()
-            self._ydl.report_warning(
-                'Writing cache to %r failed: %s' % (fn, tb))
+            self._report_warning('Writing cache to {fn!r} failed: {tb}'.format(fn=fn, tb=tb))
 
     def _validate(self, data, min_ver):
-        version = try_get(data, lambda x: x[self._VERSION_KEY])
+        version = traverse_obj(data, self._VERSION_KEY)
         if not version:  # Backward compatibility
             data, version = {'data': data}, self._DEFAULT_VERSION
         if not is_outdated_version(version, min_ver or '0', assume_new=False):
             return data['data']
-        self._ydl.to_screen(
-            'Discarding old cache from version {version} (needs {min_ver})'.format(**locals()))
+        self._write_debug('Discarding old cache from version {version} (needs {min_ver})'.format(version=version, min_ver=min_ver))
 
-    def load(self, section, key, dtype='json', default=None, min_ver=None):
+    def load(self, section, key, dtype='json', default=None, **kw_min_ver):
         assert dtype in ('json',)
+        min_ver = kw_min_ver.get('min_ver')
 
         if not self.enabled:
             return default
 
         cache_fn = self._get_cache_fn(section, key, dtype)
-        try:
+        with compat_contextlib_suppress(IOError):  # If no cache available
             try:
-                with open(cache_fn, 'r', encoding='utf-8') as cachef:
+                with open(cache_fn, encoding='utf-8') as cachef:
+                    self._write_debug('Loading {section}.{key} from cache'.format(section=section, key=key), only_once=True)
                     return self._validate(json.load(cachef), min_ver)
-            except ValueError:
+            except (ValueError, KeyError):
                 try:
                     file_size = os.path.getsize(cache_fn)
                 except (OSError, IOError) as oe:
                     file_size = error_to_compat_str(oe)
-                self._ydl.report_warning(
-                    'Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
-        except IOError:
-            pass  # No cache available
+                self._report_warning('Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
 
         return default
 
     def remove(self):
         if not self.enabled:
-            self._ydl.to_screen('Cache is disabled (Did you combine --no-cache-dir and --rm-cache-dir?)')
+            self._to_screen('Cache is disabled (Did you combine --no-cache-dir and --rm-cache-dir?)')
             return
 
         cachedir = self._get_root_dir()
         if not any((term in cachedir) for term in ('cache', 'tmp')):
-            raise Exception('Not removing directory %s - this does not look like a cache dir' % cachedir)
+            raise Exception('Not removing directory %s - this does not look like a cache dir' % (cachedir,))
 
-        self._ydl.to_screen(
-            'Removing cache dir %s .' % cachedir, skip_eol=True)
+        self._to_screen(
+            'Removing cache dir %s .' % (cachedir,), skip_eol=True, ),
         if os.path.exists(cachedir):
-            self._ydl.to_screen('.', skip_eol=True)
+            self._to_screen('.', skip_eol=True)
             shutil.rmtree(cachedir)
-        self._ydl.to_screen('.')
+        self._to_screen('.')

From 7e136639dba8a6d0e966ccc41b3dd8231587c67b Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 7 Apr 2025 15:51:03 +0100
Subject: [PATCH 842/860] [compat] Improve Py2 compatibility for URL Quoting

---
 youtube_dl/compat.py | 69 +++++++++++++++++++++++++++++++++++++-------
 1 file changed, 58 insertions(+), 11 deletions(-)

diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py
index 72b2f8f78..ebe22bdf9 100644
--- a/youtube_dl/compat.py
+++ b/youtube_dl/compat.py
@@ -2498,8 +2498,7 @@ try:
     from urllib.parse import urlencode as compat_urllib_parse_urlencode
     from urllib.parse import parse_qs as compat_parse_qs
 except ImportError:  # Python 2
-    _asciire = (compat_urllib_parse._asciire if hasattr(compat_urllib_parse, '_asciire')
-                else re.compile(r'([\x00-\x7f]+)'))
+    _asciire = getattr(compat_urllib_parse, '_asciire', None) or re.compile(r'([\x00-\x7f]+)')
 
     # HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus
     # implementations from cpython 3.4.3's stdlib. Python 2's version
@@ -2567,24 +2566,21 @@ except ImportError:  # Python 2
     # Possible solutions are to either port it from python 3 with all
     # the friends or manually ensure input query contains only byte strings.
     # We will stick with latter thus recursively encoding the whole query.
-    def compat_urllib_parse_urlencode(query, doseq=0, encoding='utf-8'):
+    def compat_urllib_parse_urlencode(query, doseq=0, safe='', encoding='utf-8', errors='strict'):
+
         def encode_elem(e):
             if isinstance(e, dict):
                 e = encode_dict(e)
             elif isinstance(e, (list, tuple,)):
-                list_e = encode_list(e)
-                e = tuple(list_e) if isinstance(e, tuple) else list_e
+                e = type(e)(encode_elem(el) for el in e)
             elif isinstance(e, compat_str):
-                e = e.encode(encoding)
+                e = e.encode(encoding, errors)
             return e
 
         def encode_dict(d):
-            return dict((encode_elem(k), encode_elem(v)) for k, v in d.items())
+            return tuple((encode_elem(k), encode_elem(v)) for k, v in d.items())
 
-        def encode_list(l):
-            return [encode_elem(e) for e in l]
-
-        return compat_urllib_parse._urlencode(encode_elem(query), doseq=doseq)
+        return compat_urllib_parse._urlencode(encode_elem(query), doseq=doseq).decode('ascii')
 
     # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib.
     # Python 2's version is apparently totally broken
@@ -2639,6 +2635,57 @@ except ImportError:  # Python 2
             ('parse_qs', compat_parse_qs)):
         setattr(compat_urllib_parse, name, fix)
 
+    try:
+        all(chr(i) in b'' for i in range(256))
+    except TypeError:
+        # not all chr(i) are str: patch Python2 quote
+
+        _safemaps = getattr(compat_urllib_parse, '_safemaps', {})
+        _always_safe = frozenset(compat_urllib_parse.always_safe)
+
+        def _quote(s, safe='/'):
+            """quote('abc def') -> 'abc%20def'"""
+
+            if not s and s is not None:  # fast path
+                return s
+            safe = frozenset(safe)
+            cachekey = (safe, _always_safe)
+            try:
+                safe_map = _safemaps[cachekey]
+            except KeyError:
+                safe = _always_safe | safe
+                safe_map = {}
+                for i in range(256):
+                    c = chr(i)
+                    safe_map[c] = (
+                        c if (i < 128 and c in safe)
+                        else b'%{0:02X}'.format(i))
+                _safemaps[cachekey] = safe_map
+
+            if safe.issuperset(s):
+                return s
+            return ''.join(safe_map[c] for c in s)
+
+        # linked code
+        def _quote_plus(s, safe=''):
+            return (
+                _quote(s, safe + b' ').replace(b' ', b'+') if b' ' in s
+                else _quote(s, safe))
+
+        # linked code
+        def _urlcleanup():
+            if compat_urllib_parse._urlopener:
+                compat_urllib_parse._urlopener.cleanup()
+            _safemaps.clear()
+            compat_urllib_parse.ftpcache.clear()
+
+        for name, fix in (
+                ('quote', _quote),
+                ('quote_plus', _quote_plus),
+                ('urlcleanup', _urlcleanup)):
+            setattr(compat_urllib_parse, '_' + name, getattr(compat_urllib_parse, name))
+            setattr(compat_urllib_parse, name, fix)
+
 compat_urllib_parse_parse_qs = compat_parse_qs
 
 

From 2190e892603d238e1a1fc40477bf30d131c22acc Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 7 Apr 2025 16:00:11 +0100
Subject: [PATCH 843/860] [utils] Support optional `safe` argument for
 `escape_rfc3986()`

---
 youtube_dl/utils.py | 8 ++++++--
 1 file changed, 6 insertions(+), 2 deletions(-)

diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py
index ac1e78002..c4262936e 100644
--- a/youtube_dl/utils.py
+++ b/youtube_dl/utils.py
@@ -4204,12 +4204,16 @@ def lowercase_escape(s):
         s)
 
 
-def escape_rfc3986(s):
+def escape_rfc3986(s, safe=None):
     """Escape non-ASCII characters as suggested by RFC 3986"""
     if sys.version_info < (3, 0):
         s = _encode_compat_str(s, 'utf-8')
+        if safe is not None:
+            safe = _encode_compat_str(safe, 'utf-8')
+    if safe is None:
+        safe = b"%/;:@&=+$,!~*'()?#[]"
     # ensure unicode: after quoting, it can always be converted
-    return compat_str(compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]"))
+    return compat_str(compat_urllib_parse.quote(s, safe))
 
 
 def escape_url(url):

From 570b8680780eba23d9e991eff99983af499f7ff0 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Mon, 7 Apr 2025 16:02:10 +0100
Subject: [PATCH 844/860] [cache] Use `esc_rfc3986` to encode cache key

---
 youtube_dl/cache.py | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/youtube_dl/cache.py b/youtube_dl/cache.py
index 6169461b9..54c24b7e4 100644
--- a/youtube_dl/cache.py
+++ b/youtube_dl/cache.py
@@ -12,10 +12,10 @@ from .compat import (
     compat_getenv,
     compat_open as open,
     compat_os_makedirs,
-    compat_urllib_parse,
 )
 from .utils import (
     error_to_compat_str,
+    escape_rfc3986,
     expand_path,
     is_outdated_version,
     traverse_obj,
@@ -55,8 +55,7 @@ class Cache(object):
     def _get_cache_fn(self, section, key, dtype):
         assert re.match(r'^[\w.-]+$', section), \
             'invalid section %r' % section
-        key = compat_urllib_parse.quote(
-            key, safe='').replace('%', ',')  # encode non-ascii characters
+        key = escape_rfc3986(key, safe='').replace('%', ',')  # encode non-ascii characters
         return os.path.join(
             self._get_root_dir(), section, '%s.%s' % (key, dtype))
 

From 32b8d317805b7f947b62af812e64789d1c5728a0 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Apr 2025 10:55:32 +0100
Subject: [PATCH 845/860] [YouTube] Support shorts playlist * only 1..100:
 yt-dlp/yt-dlp#11130

---
 youtube_dl/extractor/youtube.py | 39 ++++++++++++++++++++++++++++-----
 1 file changed, 33 insertions(+), 6 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index ce97fd75b..730e50d8e 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -3339,6 +3339,20 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                     'thumbnailViewModel', 'image'), final_key='sources'),
         })
 
+    def _extract_shorts_lockup_view_model(self, view_model):
+        content_id = traverse_obj(view_model, (
+            'onTap', 'innertubeCommand', 'reelWatchEndpoint', 'videoId',
+            T(lambda v: v if YoutubeIE.suitable(v) else None)))
+        if not content_id:
+            return
+        return merge_dicts(self.url_result(
+            content_id, ie=YoutubeIE.ie_key(), video_id=content_id), {
+                'title': traverse_obj(view_model, (
+                    'overlayMetadata', 'primaryText', 'content', T(compat_str))),
+                'thumbnails': self._extract_thumbnails(
+                    view_model, 'thumbnail', final_key='sources'),
+        })
+
     def _video_entry(self, video_renderer):
         video_id = video_renderer.get('videoId')
         if video_id:
@@ -3385,10 +3399,9 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                 yield entry
 
     def _rich_grid_entries(self, contents):
-        for content in contents:
-            content = traverse_obj(
-                content, ('richItemRenderer', 'content'),
-                expected_type=dict) or {}
+        for content in traverse_obj(
+                contents, (Ellipsis, 'richItemRenderer', 'content'),
+                expected_type=dict):
             video_renderer = traverse_obj(
                 content, 'videoRenderer', 'reelItemRenderer',
                 expected_type=dict)
@@ -3396,6 +3409,12 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                 entry = self._video_entry(video_renderer)
                 if entry:
                     yield entry
+            # shorts item
+            shorts_lockup_view_model = content.get('shortsLockupViewModel')
+            if shorts_lockup_view_model:
+                entry = self._extract_shorts_lockup_view_model(shorts_lockup_view_model)
+                if entry:
+                    yield entry
             # playlist
             renderer = traverse_obj(
                 content, 'playlistRenderer', expected_type=dict) or {}
@@ -3499,6 +3518,13 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                         entry = self._video_entry(renderer)
                         if entry:
                             yield entry
+                    renderer = isr_content.get('richGridRenderer')
+                    if renderer:
+                        for from_ in self._rich_grid_entries(
+                                traverse_obj(renderer, ('contents', Ellipsis, T(dict)))):
+                            yield from_
+                        continuation = self._extract_continuation(renderer)
+                        continue
 
                 if not continuation:
                     continuation = self._extract_continuation(is_renderer)
@@ -3508,8 +3534,9 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
             rich_grid_renderer = tab_content.get('richGridRenderer')
             if not rich_grid_renderer:
                 return
-            for entry in self._rich_grid_entries(rich_grid_renderer.get('contents') or []):
-                yield entry
+            for from_ in self._rich_grid_entries(
+                    traverse_obj(rich_grid_renderer, ('contents', Ellipsis, T(dict)))):
+                yield from_
 
             continuation = self._extract_continuation(rich_grid_renderer)
 

From 1b08d3281d3343a62007aacdc5b669c263bcebeb Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Apr 2025 10:58:01 +0100
Subject: [PATCH 846/860] [YouTube] Fix playlist continuation extraction * thx
 coletdjnz, bashonly: yt-dlp/yt-dlp#12777

---
 youtube_dl/extractor/youtube.py | 28 ++++++++++++----------------
 1 file changed, 12 insertions(+), 16 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 730e50d8e..556ef7577 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -3453,23 +3453,15 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
         next_continuation = cls._extract_next_continuation_data(renderer)
         if next_continuation:
             return next_continuation
-        contents = []
-        for key in ('contents', 'items'):
-            contents.extend(try_get(renderer, lambda x: x[key], list) or [])
-        for content in contents:
-            if not isinstance(content, dict):
-                continue
-            continuation_ep = try_get(
-                content, lambda x: x['continuationItemRenderer']['continuationEndpoint'],
-                dict)
-            if not continuation_ep:
-                continue
-            continuation = try_get(
-                continuation_ep, lambda x: x['continuationCommand']['token'], compat_str)
+        for command in traverse_obj(renderer, (
+                ('contents', 'items', 'rows'), Ellipsis, 'continuationItemRenderer',
+                ('continuationEndpoint', ('button', 'buttonRenderer', 'command')),
+                (('commandExecutorCommand', 'commands', Ellipsis), None), T(dict))):
+            continuation = traverse_obj(command, ('continuationCommand', 'token', T(compat_str)))
             if not continuation:
                 continue
-            ctp = continuation_ep.get('clickTrackingParams')
-            return YoutubeTabIE._build_continuation_query(continuation, ctp)
+            ctp = command.get('clickTrackingParams')
+            return cls._build_continuation_query(continuation, ctp)
 
     def _entries(self, tab, item_id, webpage):
         tab_content = try_get(tab, lambda x: x['content'], dict)
@@ -3582,8 +3574,12 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor):
                     # Downloading page may result in intermittent 5xx HTTP error
                     # that is usually worked around with a retry
                     response = self._download_json(
-                        'https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
+                        'https://www.youtube.com/youtubei/v1/browse',
                         None, 'Downloading page %d%s' % (page_num, ' (retry #%d)' % count if count else ''),
+                        query={
+                            # 'key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
+                            'prettyPrint': 'false',
+                        },
                         headers=headers, data=json.dumps(data).encode('utf8'))
                     break
                 except ExtractorError as e:

From 6c40d9f8472b5e98ca3a3880cfce01a8403f3a9f Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Apr 2025 11:10:29 +0100
Subject: [PATCH 847/860] [YouTube] Remove remaining hard-coded API keys * no
 longer required for these cases

---
 youtube_dl/extractor/youtube.py | 15 ++++++---------
 1 file changed, 6 insertions(+), 9 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 556ef7577..7f8c0c532 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -342,14 +342,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
         if not self._login():
             return
 
-    _DEFAULT_API_DATA = {
-        'context': {
-            'client': {
-                'clientName': 'WEB',
-                'clientVersion': '2.20201021.03.00',
-            },
-        },
-    }
+    _DEFAULT_API_DATA = {'context': _INNERTUBE_CLIENTS['web']['INNERTUBE_CONTEXT']}
 
     _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*=\s*({.+?})\s*;'
     _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;'
@@ -497,11 +490,15 @@ class YoutubeBaseInfoExtractor(InfoExtractor):
             data['params'] = params
         for page_num in itertools.count(1):
             search = self._download_json(
-                'https://www.youtube.com/youtubei/v1/search?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
+                'https://www.youtube.com/youtubei/v1/search',
                 video_id='query "%s"' % query,
                 note='Downloading page %s' % page_num,
                 errnote='Unable to download API page', fatal=False,
                 data=json.dumps(data).encode('utf8'),
+                query={
+                    # 'key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8',
+                    'prettyPrint': 'false',
+                },
                 headers={'content-type': 'application/json'})
             if not search:
                 break

From 6eb6d6dff5add8b17babebb3a1a3da7948e5ee18 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Apr 2025 11:36:35 +0100
Subject: [PATCH 848/860] [InfoExtractor] Use local variants for remaining
 parent method calls * ... where defined

---
 youtube_dl/cache.py            |  6 +--
 youtube_dl/extractor/common.py | 70 +++++++++++++++++-----------------
 2 files changed, 38 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/cache.py b/youtube_dl/cache.py
index 54c24b7e4..ba12d0373 100644
--- a/youtube_dl/cache.py
+++ b/youtube_dl/cache.py
@@ -42,11 +42,11 @@ class Cache(object):
     def _to_screen(self, *args, **kwargs):
         self._ydl.to_screen(*args, **kwargs)
 
-    def _get_params(self, k, default=None):
+    def _get_param(self, k, default=None):
         return self._ydl.params.get(k, default)
 
     def _get_root_dir(self):
-        res = self._get_params('cachedir')
+        res = self._get_param('cachedir')
         if res is None:
             cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache')
             res = os.path.join(cache_root, self._YTDL_DIR)
@@ -61,7 +61,7 @@ class Cache(object):
 
     @property
     def enabled(self):
-        return self._get_params('cachedir') is not False
+        return self._get_param('cachedir') is not False
 
     def store(self, section, key, data, dtype='json'):
         assert dtype in ('json',)
diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index cbc4e968a..6f4a48bcd 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -505,7 +505,7 @@ class InfoExtractor(object):
         if not self._x_forwarded_for_ip:
 
             # Geo bypass mechanism is explicitly disabled by user
-            if not self._downloader.params.get('geo_bypass', True):
+            if not self.get_param('geo_bypass', True):
                 return
 
             if not geo_bypass_context:
@@ -527,7 +527,7 @@ class InfoExtractor(object):
 
             # Explicit IP block specified by user, use it right away
             # regardless of whether extractor is geo bypassable or not
-            ip_block = self._downloader.params.get('geo_bypass_ip_block', None)
+            ip_block = self.get_param('geo_bypass_ip_block', None)
 
             # Otherwise use random IP block from geo bypass context but only
             # if extractor is known as geo bypassable
@@ -538,8 +538,8 @@ class InfoExtractor(object):
 
             if ip_block:
                 self._x_forwarded_for_ip = GeoUtils.random_ipv4(ip_block)
-                if self._downloader.params.get('verbose', False):
-                    self._downloader.to_screen(
+                if self.get_param('verbose', False):
+                    self.to_screen(
                         '[debug] Using fake IP %s as X-Forwarded-For.'
                         % self._x_forwarded_for_ip)
                 return
@@ -548,7 +548,7 @@ class InfoExtractor(object):
 
             # Explicit country code specified by user, use it right away
             # regardless of whether extractor is geo bypassable or not
-            country = self._downloader.params.get('geo_bypass_country', None)
+            country = self.get_param('geo_bypass_country', None)
 
             # Otherwise use random country code from geo bypass context but
             # only if extractor is known as geo bypassable
@@ -559,8 +559,8 @@ class InfoExtractor(object):
 
             if country:
                 self._x_forwarded_for_ip = GeoUtils.random_ipv4(country)
-                if self._downloader.params.get('verbose', False):
-                    self._downloader.to_screen(
+                if self.get_param('verbose', False):
+                    self.to_screen(
                         '[debug] Using fake IP %s (%s) as X-Forwarded-For.'
                         % (self._x_forwarded_for_ip, country.upper()))
 
@@ -586,9 +586,9 @@ class InfoExtractor(object):
             raise ExtractorError('An extractor error has occurred.', cause=e)
 
     def __maybe_fake_ip_and_retry(self, countries):
-        if (not self._downloader.params.get('geo_bypass_country', None)
+        if (not self.get_param('geo_bypass_country', None)
                 and self._GEO_BYPASS
-                and self._downloader.params.get('geo_bypass', True)
+                and self.get_param('geo_bypass', True)
                 and not self._x_forwarded_for_ip
                 and countries):
             country_code = random.choice(countries)
@@ -698,7 +698,7 @@ class InfoExtractor(object):
             if fatal:
                 raise ExtractorError(errmsg, sys.exc_info()[2], cause=err)
             else:
-                self._downloader.report_warning(errmsg)
+                self.report_warning(errmsg)
                 return False
 
     def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None, data=None, headers={}, query={}, expected_status=None):
@@ -770,11 +770,11 @@ class InfoExtractor(object):
             webpage_bytes = prefix + webpage_bytes
         if not encoding:
             encoding = self._guess_encoding_from_content(content_type, webpage_bytes)
-        if self._downloader.params.get('dump_intermediate_pages', False):
+        if self.get_param('dump_intermediate_pages', False):
             self.to_screen('Dumping request to ' + urlh.geturl())
             dump = base64.b64encode(webpage_bytes).decode('ascii')
-            self._downloader.to_screen(dump)
-        if self._downloader.params.get('write_pages', False):
+            self.to_screen(dump)
+        if self.get_param('write_pages', False):
             basen = '%s_%s' % (video_id, urlh.geturl())
             if len(basen) > 240:
                 h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest()
@@ -1074,7 +1074,7 @@ class InfoExtractor(object):
                 if mobj:
                     break
 
-        if not self._downloader.params.get('no_color') and compat_os_name != 'nt' and sys.stderr.isatty():
+        if not self.get_param('no_color') and compat_os_name != 'nt' and sys.stderr.isatty():
             _name = '\033[0;34m%s\033[0m' % name
         else:
             _name = name
@@ -1092,7 +1092,7 @@ class InfoExtractor(object):
         elif fatal:
             raise RegexNotFoundError('Unable to extract %s' % _name)
         else:
-            self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message())
+            self.report_warning('unable to extract %s' % _name + bug_reports_message())
             return None
 
     def _search_json(self, start_pattern, string, name, video_id, **kwargs):
@@ -1162,7 +1162,7 @@ class InfoExtractor(object):
         username = None
         password = None
 
-        if self._downloader.params.get('usenetrc', False):
+        if self.get_param('usenetrc', False):
             try:
                 netrc_machine = netrc_machine or self._NETRC_MACHINE
                 info = netrc.netrc().authenticators(netrc_machine)
@@ -1173,7 +1173,7 @@ class InfoExtractor(object):
                     raise netrc.NetrcParseError(
                         'No authenticators for %s' % netrc_machine)
             except (AttributeError, IOError, netrc.NetrcParseError) as err:
-                self._downloader.report_warning(
+                self.report_warning(
                     'parsing .netrc: %s' % error_to_compat_str(err))
 
         return username, password
@@ -1210,10 +1210,10 @@ class InfoExtractor(object):
         """
         if self._downloader is None:
             return None
-        downloader_params = self._downloader.params
 
-        if downloader_params.get('twofactor') is not None:
-            return downloader_params['twofactor']
+        twofactor = self.get_param('twofactor')
+        if twofactor is not None:
+            return twofactor
 
         return compat_getpass('Type %s and press [Return]: ' % note)
 
@@ -1348,7 +1348,7 @@ class InfoExtractor(object):
         elif fatal:
             raise RegexNotFoundError('Unable to extract JSON-LD')
         else:
-            self._downloader.report_warning('unable to extract JSON-LD %s' % bug_reports_message())
+            self.report_warning('unable to extract JSON-LD %s' % bug_reports_message())
             return {}
 
     def _json_ld(self, json_ld, video_id, fatal=True, expected_type=None):
@@ -1579,7 +1579,7 @@ class InfoExtractor(object):
 
             if f.get('vcodec') == 'none':  # audio only
                 preference -= 50
-                if self._downloader.params.get('prefer_free_formats'):
+                if self.get_param('prefer_free_formats'):
                     ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus']
                 else:
                     ORDER = ['webm', 'opus', 'ogg', 'mp3', 'aac', 'm4a']
@@ -1591,7 +1591,7 @@ class InfoExtractor(object):
             else:
                 if f.get('acodec') == 'none':  # video only
                     preference -= 40
-                if self._downloader.params.get('prefer_free_formats'):
+                if self.get_param('prefer_free_formats'):
                     ORDER = ['flv', 'mp4', 'webm']
                 else:
                     ORDER = ['webm', 'flv', 'mp4']
@@ -1657,7 +1657,7 @@ class InfoExtractor(object):
         """ Either "http:" or "https:", depending on the user's preferences """
         return (
             'http:'
-            if self._downloader.params.get('prefer_insecure', False)
+            if self.get_param('prefer_insecure', False)
             else 'https:')
 
     def _proto_relative_url(self, url, scheme=None):
@@ -3189,7 +3189,7 @@ class InfoExtractor(object):
             if fatal:
                 raise ExtractorError(msg)
             else:
-                self._downloader.report_warning(msg)
+                self.report_warning(msg)
         return res
 
     def _float(self, v, name, fatal=False, **kwargs):
@@ -3199,7 +3199,7 @@ class InfoExtractor(object):
             if fatal:
                 raise ExtractorError(msg)
             else:
-                self._downloader.report_warning(msg)
+                self.report_warning(msg)
         return res
 
     def _set_cookie(self, domain, name, value, expire_time=None, port=None,
@@ -3208,12 +3208,12 @@ class InfoExtractor(object):
             0, name, value, port, port is not None, domain, True,
             domain.startswith('.'), path, True, secure, expire_time,
             discard, None, None, rest)
-        self._downloader.cookiejar.set_cookie(cookie)
+        self.cookiejar.set_cookie(cookie)
 
     def _get_cookies(self, url):
         """ Return a compat_cookies_SimpleCookie with the cookies for the url """
         req = sanitized_Request(url)
-        self._downloader.cookiejar.add_cookie_header(req)
+        self.cookiejar.add_cookie_header(req)
         return compat_cookies_SimpleCookie(req.get_header('Cookie'))
 
     def _apply_first_set_cookie_header(self, url_handle, cookie):
@@ -3273,8 +3273,8 @@ class InfoExtractor(object):
         return not any_restricted
 
     def extract_subtitles(self, *args, **kwargs):
-        if (self._downloader.params.get('writesubtitles', False)
-                or self._downloader.params.get('listsubtitles')):
+        if (self.get_param('writesubtitles', False)
+                or self.get_param('listsubtitles')):
             return self._get_subtitles(*args, **kwargs)
         return {}
 
@@ -3303,8 +3303,8 @@ class InfoExtractor(object):
         return target
 
     def extract_automatic_captions(self, *args, **kwargs):
-        if (self._downloader.params.get('writeautomaticsub', False)
-                or self._downloader.params.get('listsubtitles')):
+        if (self.get_param('writeautomaticsub', False)
+                or self.get_param('listsubtitles')):
             return self._get_automatic_captions(*args, **kwargs)
         return {}
 
@@ -3312,9 +3312,9 @@ class InfoExtractor(object):
         raise NotImplementedError('This method must be implemented by subclasses')
 
     def mark_watched(self, *args, **kwargs):
-        if (self._downloader.params.get('mark_watched', False)
+        if (self.get_param('mark_watched', False)
                 and (self._get_login_info()[0] is not None
-                     or self._downloader.params.get('cookiefile') is not None)):
+                     or self.get_param('cookiefile') is not None)):
             self._mark_watched(*args, **kwargs)
 
     def _mark_watched(self, *args, **kwargs):
@@ -3322,7 +3322,7 @@ class InfoExtractor(object):
 
     def geo_verification_headers(self):
         headers = {}
-        geo_verification_proxy = self._downloader.params.get('geo_verification_proxy')
+        geo_verification_proxy = self.get_param('geo_verification_proxy')
         if geo_verification_proxy:
             headers['Ytdl-request-proxy'] = geo_verification_proxy
         return headers

From 67dbfa65f220e358de713977bed8d818059f7f27 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Apr 2025 11:43:32 +0100
Subject: [PATCH 849/860] [InfoExtractor] Fix merging subtitles to empty target

---
 youtube_dl/extractor/common.py | 6 +++++-
 1 file changed, 5 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py
index 6f4a48bcd..a64fcfccc 100644
--- a/youtube_dl/extractor/common.py
+++ b/youtube_dl/extractor/common.py
@@ -3295,7 +3295,11 @@ class InfoExtractor(object):
         """ Merge subtitle dictionaries, language by language. """
 
         # ..., * , target=None
-        target = kwargs.get('target') or dict(subtitle_dict1)
+        target = kwargs.get('target')
+        if target is None:
+            target = dict(subtitle_dict1)
+        else:
+            subtitle_dicts = (subtitle_dict1,) + subtitle_dicts
 
         for subtitle_dict in subtitle_dicts:
             for lang in subtitle_dict:

From 75134137947d1cd58b73f4186b3693d032c5bb66 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Apr 2025 12:15:38 +0100
Subject: [PATCH 850/860] [JSInterp] Reorganise some declarations to align
 better with yt-dlp

---
 test/test_jsinterp.py  | 39 ++++++++++++++++++++-------------------
 youtube_dl/jsinterp.py | 22 +++++++++++-----------
 2 files changed, 31 insertions(+), 30 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 3c9650ab6..b0ac3a704 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -7,6 +7,7 @@ from __future__ import unicode_literals
 import os
 import sys
 import unittest
+
 sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
 
 import math
@@ -146,6 +147,25 @@ class TestJSInterpreter(unittest.TestCase):
         # https://github.com/ytdl-org/youtube-dl/issues/32815
         self._test('function f(){return 0  - 7 * - 6;}', 42)
 
+    def test_bitwise_operators_typecast(self):
+        # madness
+        self._test('function f(){return null << 5}', 0)
+        self._test('function f(){return undefined >> 5}', 0)
+        self._test('function f(){return 42 << NaN}', 42)
+        self._test('function f(){return 42 << Infinity}', 42)
+        self._test('function f(){return 0.0 << null}', 0)
+        self._test('function f(){return NaN << 42}', 0)
+        self._test('function f(){return "21.9" << 1}', 42)
+        self._test('function f(){return true << "5";}', 32)
+        self._test('function f(){return true << true;}', 2)
+        self._test('function f(){return "19" & "21.9";}', 17)
+        self._test('function f(){return "19" & false;}', 0)
+        self._test('function f(){return "11.0" >> "2.1";}', 2)
+        self._test('function f(){return 5 ^ 9;}', 12)
+        self._test('function f(){return 0.0 << NaN}', 0)
+        self._test('function f(){return null << undefined}', 0)
+        self._test('function f(){return 21 << 4294967297}', 42)
+
     def test_array_access(self):
         self._test('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}', [5, 2, 7])
 
@@ -482,25 +502,6 @@ class TestJSInterpreter(unittest.TestCase):
         self._test('function f(){return -524999584 << 5}', 379882496)
         self._test('function f(){return 1236566549 << 5}', 915423904)
 
-    def test_bitwise_operators_typecast(self):
-        # madness
-        self._test('function f(){return null << 5}', 0)
-        self._test('function f(){return undefined >> 5}', 0)
-        self._test('function f(){return 42 << NaN}', 42)
-        self._test('function f(){return 42 << Infinity}', 42)
-        self._test('function f(){return 0.0 << null}', 0)
-        self._test('function f(){return NaN << 42}', 0)
-        self._test('function f(){return "21.9" << 1}', 42)
-        self._test('function f(){return 21 << 4294967297}', 42)
-        self._test('function f(){return true << "5";}', 32)
-        self._test('function f(){return true << true;}', 2)
-        self._test('function f(){return "19" & "21.9";}', 17)
-        self._test('function f(){return "19" & false;}', 0)
-        self._test('function f(){return "11.0" >> "2.1";}', 2)
-        self._test('function f(){return 5 ^ 9;}', 12)
-        self._test('function f(){return 0.0 << NaN}', 0)
-        self._test('function f(){return null << undefined}', 0)
-
     def test_negative(self):
         self._test('function f(){return 2    *    -2.0    ;}', -4)
         self._test('function f(){return 2    -    - -2    ;}', 0)
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 69c8f77ca..971387df2 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -283,17 +283,6 @@ _OPERATORS = (
     ('**', _js_exp),
 )
 
-_COMP_OPERATORS = (
-    ('===', _js_id_op(operator.is_)),
-    ('!==', _js_id_op(operator.is_not)),
-    ('==', _js_eq),
-    ('!=', _js_neq),
-    ('<=', _js_comp_op(operator.le)),
-    ('>=', _js_comp_op(operator.ge)),
-    ('<', _js_comp_op(operator.lt)),
-    ('>', _js_comp_op(operator.gt)),
-)
-
 _LOG_OPERATORS = (
     ('|', _js_bit_op(operator.or_)),
     ('^', _js_bit_op(operator.xor)),
@@ -314,6 +303,17 @@ _UNARY_OPERATORS_X = (
 
 _OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS))
 
+_COMP_OPERATORS = (
+    ('===', _js_id_op(operator.is_)),
+    ('!==', _js_id_op(operator.is_not)),
+    ('==', _js_eq),
+    ('!=', _js_neq),
+    ('<=', _js_comp_op(operator.le)),
+    ('>=', _js_comp_op(operator.ge)),
+    ('<', _js_comp_op(operator.lt)),
+    ('>', _js_comp_op(operator.gt)),
+)
+
 _NAME_RE = r'[a-zA-Z_$][\w$]*'
 _MATCHING_PARENS = dict(zip(*zip('()', '{}', '[]')))
 _QUOTES = '\'"/'

From d21717978cff5321ca279593340051d4bc73c129 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Apr 2025 12:20:15 +0100
Subject: [PATCH 851/860] [JSInterp] Improve JS classes, etc

---
 test/test_jsinterp.py  |  1 +
 youtube_dl/jsinterp.py | 85 ++++++++++++++++++++++++++++++++++--------
 2 files changed, 71 insertions(+), 15 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index b0ac3a704..30c48adfc 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -455,6 +455,7 @@ class TestJSInterpreter(unittest.TestCase):
 
     def test_regex(self):
         self._test('function f() { let a=/,,[/,913,/](,)}/; }', None)
+        self._test('function f() { let a=/,,[/,913,/](,)}/; return a.source;  }', ',,[/,913,/](,)}')
 
         jsi = JSInterpreter('''
             function x() { let a=/,,[/,913,/](,)}/; "".replace(a, ""); return a; }
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 971387df2..4a464743f 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -353,7 +353,7 @@ class LocalNameSpace(ChainMap):
         raise NotImplementedError('Deleting is not supported')
 
     def __repr__(self):
-        return 'LocalNameSpace%s' % (self.maps, )
+        return 'LocalNameSpace({0!r})'.format(self.maps)
 
 
 class Debugger(object):
@@ -374,6 +374,9 @@ class Debugger(object):
 
     @classmethod
     def wrap_interpreter(cls, f):
+        if not cls.ENABLED:
+            return f
+
         @wraps(f)
         def interpret_statement(self, stmt, local_vars, allow_recursion, *args, **kwargs):
             if cls.ENABLED and stmt.strip():
@@ -414,7 +417,17 @@ class JSInterpreter(object):
                 msg = '{0} in: {1!r:.100}'.format(msg.rstrip(), expr)
             super(JSInterpreter.Exception, self).__init__(msg, *args, **kwargs)
 
-    class JS_RegExp(object):
+    class JS_Object(object):
+        def __getitem__(self, key):
+            if hasattr(self, key):
+                return getattr(self, key)
+            raise KeyError(key)
+
+        def dump(self):
+            """Serialise the instance"""
+            raise NotImplementedError
+
+    class JS_RegExp(JS_Object):
         RE_FLAGS = {
             # special knowledge: Python's re flags are bitmask values, current max 128
             # invent new bitmask values well above that for literal parsing
@@ -435,16 +448,24 @@ class JSInterpreter(object):
         def __init__(self, pattern_txt, flags=0):
             if isinstance(flags, compat_str):
                 flags, _ = self.regex_flags(flags)
-            # First, avoid https://github.com/python/cpython/issues/74534
             self.__self = None
             pattern_txt = str_or_none(pattern_txt) or '(?:)'
-            self.__pattern_txt = pattern_txt.replace('[[', r'[\[')
+            # escape unintended embedded flags
+            pattern_txt = re.sub(
+                r'(\(\?)([aiLmsux]*)(-[imsx]+:|(?<!\?)\))',
+                lambda m: ''.join(
+                    (re.escape(m.group(1)), m.group(2), re.escape(m.group(3)))
+                    if m.group(3) == ')'
+                    else ('(?:', m.group(2), m.group(3))),
+                pattern_txt)
+            # Avoid https://github.com/python/cpython/issues/74534
+            self.source = pattern_txt.replace('[[', r'[\[')
             self.__flags = flags
 
         def __instantiate(self):
             if self.__self:
                 return
-            self.__self = re.compile(self.__pattern_txt, self.__flags)
+            self.__self = re.compile(self.source, self.__flags)
             # Thx: https://stackoverflow.com/questions/44773522/setattr-on-python2-sre-sre-pattern
             for name in dir(self.__self):
                 # Only these? Obviously __class__, __init__.
@@ -452,16 +473,15 @@ class JSInterpreter(object):
                 # that can't be setattr'd but also can't need to be copied.
                 if name in ('__class__', '__init__', '__weakref__'):
                     continue
-                setattr(self, name, getattr(self.__self, name))
+                if name == 'flags':
+                    setattr(self, name, getattr(self.__self, name, self.__flags))
+                else:
+                    setattr(self, name, getattr(self.__self, name))
 
         def __getattr__(self, name):
             self.__instantiate()
-            # make Py 2.6 conform to its lying documentation
-            if name == 'flags':
-                self.flags = self.__flags
-                return self.flags
-            elif name == 'pattern':
-                self.pattern = self.__pattern_txt
+            if name == 'pattern':
+                self.pattern = self.source
                 return self.pattern
             elif hasattr(self.__self, name):
                 v = getattr(self.__self, name)
@@ -469,6 +489,26 @@ class JSInterpreter(object):
                 return v
             elif name in ('groupindex', 'groups'):
                 return 0 if name == 'groupindex' else {}
+            else:
+                flag_attrs = (  # order by 2nd elt
+                    ('hasIndices', 'd'),
+                    ('global', 'g'),
+                    ('ignoreCase', 'i'),
+                    ('multiline', 'm'),
+                    ('dotAll', 's'),
+                    ('unicode', 'u'),
+                    ('unicodeSets', 'v'),
+                    ('sticky', 'y'),
+                )
+                for k, c in flag_attrs:
+                    if name == k:
+                        return bool(self.RE_FLAGS[c] & self.__flags)
+                else:
+                    if name == 'flags':
+                        return ''.join(
+                            (c if self.RE_FLAGS[c] & self.__flags else '')
+                            for _, c in flag_attrs)
+
             raise AttributeError('{0} has no attribute named {1}'.format(self, name))
 
         @classmethod
@@ -482,7 +522,16 @@ class JSInterpreter(object):
                 flags |= cls.RE_FLAGS[ch]
             return flags, expr[idx + 1:]
 
-    class JS_Date(object):
+        def dump(self):
+            return '(/{0}/{1})'.format(
+                re.sub(r'(?<!\\)/', r'\/', self.source),
+                self.flags)
+
+        @staticmethod
+        def escape(string_):
+            return re.escape(string_)
+
+    class JS_Date(JS_Object):
         _t = None
 
         @staticmethod
@@ -549,6 +598,9 @@ class JSInterpreter(object):
         def valueOf(self):
             return _NaN if self._t is None else self._t
 
+        def dump(self):
+            return '(new Date({0}))'.format(self.toString())
+
     @classmethod
     def __op_chars(cls):
         op_chars = set(';,[')
@@ -1109,13 +1161,15 @@ class JSInterpreter(object):
             def eval_method(variable, member):
                 if (variable, member) == ('console', 'debug'):
                     if Debugger.ENABLED:
-                        Debugger.write(self.interpret_expression('[{}]'.format(arg_str), local_vars, allow_recursion))
+                        Debugger.write(self.interpret_expression('[{0}]'.format(arg_str), local_vars, allow_recursion))
                     return
                 types = {
                     'String': compat_str,
                     'Math': float,
                     'Array': list,
                     'Date': self.JS_Date,
+                    'RegExp': self.JS_RegExp,
+                    # 'Error': self.Exception,  # has no std static methods
                 }
                 obj = local_vars.get(variable)
                 if obj in (JS_Undefined, None):
@@ -1277,7 +1331,8 @@ class JSInterpreter(object):
                     assertion(len(argvals) == 2, 'takes exactly two arguments')
                     # TODO: argvals[1] callable, other Py vs JS edge cases
                     if isinstance(argvals[0], self.JS_RegExp):
-                        count = 0 if argvals[0].flags & self.JS_RegExp.RE_FLAGS['g'] else 1
+                        # access JS member with Py reserved name
+                        count = 0 if self._index(argvals[0], 'global') else 1
                         assertion(member != 'replaceAll' or count == 0,
                                   'replaceAll must be called with a global RegExp')
                         return argvals[0].sub(argvals[1], obj, count=count)

From 16b7e97afad2ed73c2d01a55987cb323407ca1c4 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Apr 2025 12:25:17 +0100
Subject: [PATCH 852/860] [JSInterp] Add `_separate_at_op()`

---
 youtube_dl/jsinterp.py | 126 ++++++++++++++++++++++++-----------------
 1 file changed, 75 insertions(+), 51 deletions(-)

diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 4a464743f..3824a0f6b 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -704,6 +704,68 @@ class JSInterpreter(object):
                 _SC_OPERATORS, _LOG_OPERATORS, _COMP_OPERATORS, _OPERATORS, _UNARY_OPERATORS_X))
         return _cached
 
+    def _separate_at_op(self, expr, max_split=None):
+
+        for op, _ in self._all_operators():
+            # hackety: </> have higher priority than <</>>, but don't confuse them
+            skip_delim = (op + op) if op in '<>*?' else None
+            if op == '?':
+                skip_delim = (skip_delim, '?.')
+            separated = list(self._separate(expr, op, skip_delims=skip_delim))
+            if len(separated) < 2:
+                continue
+
+            right_expr = separated.pop()
+            # handle operators that are both unary and binary, minimal BODMAS
+            if op in ('+', '-'):
+                # simplify/adjust consecutive instances of these operators
+                undone = 0
+                separated = [s.strip() for s in separated]
+                while len(separated) > 1 and not separated[-1]:
+                    undone += 1
+                    separated.pop()
+                if op == '-' and undone % 2 != 0:
+                    right_expr = op + right_expr
+                elif op == '+':
+                    while len(separated) > 1 and set(separated[-1]) <= self.OP_CHARS:
+                        right_expr = separated.pop() + right_expr
+                    if separated[-1][-1:] in self.OP_CHARS:
+                        right_expr = separated.pop() + right_expr
+                # hanging op at end of left => unary + (strip) or - (push right)
+                separated.append(right_expr)
+                dm_ops = ('*', '%', '/', '**')
+                dm_chars = set(''.join(dm_ops))
+
+                def yield_terms(s):
+                    skip = False
+                    for i, term in enumerate(s[:-1]):
+                        if skip:
+                            skip = False
+                            continue
+                        if not (dm_chars & set(term)):
+                            yield term
+                            continue
+                        for dm_op in dm_ops:
+                            bodmas = list(self._separate(term, dm_op, skip_delims=skip_delim))
+                            if len(bodmas) > 1 and not bodmas[-1].strip():
+                                bodmas[-1] = (op if op == '-' else '') + s[i + 1]
+                                yield dm_op.join(bodmas)
+                                skip = True
+                                break
+                        else:
+                            if term:
+                                yield term
+
+                    if not skip and s[-1]:
+                        yield s[-1]
+
+                separated = list(yield_terms(separated))
+                right_expr = separated.pop() if len(separated) > 1 else None
+                expr = op.join(separated)
+            if right_expr is None:
+                continue
+            return op, separated, right_expr
+
     def _operator(self, op, left_val, right_expr, expr, local_vars, allow_recursion):
         if op in ('||', '&&'):
             if (op == '&&') ^ _js_ternary(left_val):
@@ -759,51 +821,9 @@ class JSInterpreter(object):
     _FINALLY_RE = re.compile(r'finally\s*\{')
     _SWITCH_RE = re.compile(r'switch\s*\(')
 
-    def handle_operators(self, expr, local_vars, allow_recursion):
-
-        for op, _ in self._all_operators():
-            # hackety: </> have higher priority than <</>>, but don't confuse them
-            skip_delim = (op + op) if op in '<>*?' else None
-            if op == '?':
-                skip_delim = (skip_delim, '?.')
-            separated = list(self._separate(expr, op, skip_delims=skip_delim))
-            if len(separated) < 2:
-                continue
-
-            right_expr = separated.pop()
-            # handle operators that are both unary and binary, minimal BODMAS
-            if op in ('+', '-'):
-                # simplify/adjust consecutive instances of these operators
-                undone = 0
-                separated = [s.strip() for s in separated]
-                while len(separated) > 1 and not separated[-1]:
-                    undone += 1
-                    separated.pop()
-                if op == '-' and undone % 2 != 0:
-                    right_expr = op + right_expr
-                elif op == '+':
-                    while len(separated) > 1 and set(separated[-1]) <= self.OP_CHARS:
-                        right_expr = separated.pop() + right_expr
-                    if separated[-1][-1:] in self.OP_CHARS:
-                        right_expr = separated.pop() + right_expr
-                # hanging op at end of left => unary + (strip) or - (push right)
-                left_val = separated[-1] if separated else ''
-                for dm_op in ('*', '%', '/', '**'):
-                    bodmas = tuple(self._separate(left_val, dm_op, skip_delims=skip_delim))
-                    if len(bodmas) > 1 and not bodmas[-1].strip():
-                        expr = op.join(separated) + op + right_expr
-                        if len(separated) > 1:
-                            separated.pop()
-                            right_expr = op.join((left_val, right_expr))
-                        else:
-                            separated = [op.join((left_val, right_expr))]
-                            right_expr = None
-                        break
-                if right_expr is None:
-                    continue
-
-            left_val = self.interpret_expression(op.join(separated), local_vars, allow_recursion)
-            return self._operator(op, left_val, right_expr, expr, local_vars, allow_recursion), True
+    def _eval_operator(self, op, left_expr, right_expr, expr, local_vars, allow_recursion):
+        left_val = self.interpret_expression(left_expr, local_vars, allow_recursion)
+        return self._operator(op, left_val, right_expr, expr, local_vars, allow_recursion)
 
     @Debugger.wrap_interpreter
     def interpret_statement(self, stmt, local_vars, allow_recursion=100):
@@ -865,9 +885,12 @@ class JSInterpreter(object):
             operand = expr[len(op):]
             if not operand or operand[0] != ' ':
                 continue
-            op_result = self.handle_operators(expr, local_vars, allow_recursion)
-            if op_result:
-                return op_result[0], should_return
+            separated = self._separate_at_op(operand, max_split=1)
+            if separated:
+                next_op, separated, right_expr = separated
+                separated.append(right_expr)
+                operand = next_op.join(separated)
+            return self._eval_operator(op, operand, '', expr, local_vars, allow_recursion), should_return
 
         if expr.startswith('{'):
             inner, outer = self._separate_at_paren(expr)
@@ -1138,9 +1161,10 @@ class JSInterpreter(object):
                 val = self._index(val, idx)
             return val, should_return
 
-        op_result = self.handle_operators(expr, local_vars, allow_recursion)
-        if op_result:
-            return op_result[0], should_return
+        separated = self._separate_at_op(expr)
+        if separated:
+            op, separated, right_expr = separated
+            return self._eval_operator(op, op.join(separated), right_expr, expr, local_vars, allow_recursion), should_return
 
         if md.get('attribute'):
             variable, member, nullish = m.group('var', 'member', 'nullish')

From bd2ded59f218bed637514f3aaf11787fa7b87ccf Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Apr 2025 12:29:28 +0100
Subject: [PATCH 853/860] [JSInterp] Improve unary operators; add `!`

---
 test/test_jsinterp.py  | 7 +++++++
 youtube_dl/jsinterp.py | 9 ++++++---
 2 files changed, 13 insertions(+), 3 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 30c48adfc..95bac75d7 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -371,6 +371,13 @@ class TestJSInterpreter(unittest.TestCase):
         self._test('function f() { a=5; return (a -= 1, a+=3, a); }', 7)
         self._test('function f() { return (l=[0,1,2,3], function(a, b){return a+b})((l[1], l[2]), l[3]) }', 5)
 
+    def test_not(self):
+        self._test('function f() { return ! undefined; }', True)
+        self._test('function f() { return !0; }', True)
+        self._test('function f() { return !!0; }', False)
+        self._test('function f() { return ![]; }', False)
+        self._test('function f() { return !0 !== false; }', True)
+
     def test_void(self):
         self._test('function f() { return void 42; }', JS_Undefined)
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index 3824a0f6b..ff9894f52 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -240,7 +240,7 @@ def _js_ternary(cndn, if_true=True, if_false=False):
 def _js_unary_op(op):
 
     @wraps_op(op)
-    def wrapped(_, a):
+    def wrapped(a, _):
         return op(a)
 
     return wrapped
@@ -299,6 +299,8 @@ _SC_OPERATORS = (
 _UNARY_OPERATORS_X = (
     ('void', _js_unary_op(lambda _: JS_Undefined)),
     ('typeof', _js_unary_op(_js_typeof)),
+    # avoid functools.partial here since Py2 update_wrapper(partial) -> no __module__
+    ('!', _js_unary_op(lambda x: _js_ternary(x, if_true=False, if_false=True))),
 )
 
 _OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS))
@@ -776,7 +778,7 @@ class JSInterpreter(object):
         elif op == '?':
             right_expr = _js_ternary(left_val, *self._separate(right_expr, ':', 1))
 
-        right_val = self.interpret_expression(right_expr, local_vars, allow_recursion)
+        right_val = self.interpret_expression(right_expr, local_vars, allow_recursion) if right_expr else left_val
         opfunc = op and next((v for k, v in self._all_operators() if k == op), None)
         if not opfunc:
             return right_val
@@ -879,11 +881,12 @@ class JSInterpreter(object):
             else:
                 raise self.Exception('Unsupported object {obj:.100}'.format(**locals()), expr=expr)
 
+        # apply unary operators (see new above)
         for op, _ in _UNARY_OPERATORS_X:
             if not expr.startswith(op):
                 continue
             operand = expr[len(op):]
-            if not operand or operand[0] != ' ':
+            if not operand or (op.isalpha() and operand[0] != ' '):
                 continue
             separated = self._separate_at_op(operand, max_split=1)
             if separated:

From c052a16f72af7dd7671d4dd62826de71cd99dfb6 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 4 Apr 2025 12:53:28 +0100
Subject: [PATCH 854/860] [JSInterp] Add tests and relevant functionality from
 yt-dlp * thx seproDev, bashonly: yt-dlp/yt-dlp#12760, yt-dlp/yt-dlp#12761:  
 - Improve nested attribute support   - Pass global stack when extracting
 objects   - interpret_statement: Match attribute before indexing   - Fix
 assignment to array elements with nested brackets   - Add new signature tests
   - Invalidate JS function cache   - Avoid testdata dupes now that we cache
 by URL

* rework nsig function name search
* fully fixes #33102
* update cache required versions
* update program version
---
 test/test_jsinterp.py           | 16 +++++++++
 test/test_youtube_signature.py  | 64 +++++++++++++++++++++++++++++++--
 youtube_dl/extractor/youtube.py | 33 ++++++++++++++---
 youtube_dl/jsinterp.py          | 36 +++++++++++--------
 youtube_dl/version.py           |  2 +-
 5 files changed, 129 insertions(+), 22 deletions(-)

diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py
index 95bac75d7..479cb43a0 100644
--- a/test/test_jsinterp.py
+++ b/test/test_jsinterp.py
@@ -180,6 +180,7 @@ class TestJSInterpreter(unittest.TestCase):
         self._test('function f(){var x = 20; x = 30 + 1; return x;}', 31)
         self._test('function f(){var x = 20; x += 30 + 1; return x;}', 51)
         self._test('function f(){var x = 20; x -= 30 + 1; return x;}', -11)
+        self._test('function f(){var x = 2; var y = ["a", "b"]; y[x%y["length"]]="z"; return y}', ['z', 'b'])
 
     def test_comments(self):
         self._test('''
@@ -552,6 +553,8 @@ class TestJSInterpreter(unittest.TestCase):
         test_result = list('test')
         tests = [
             'function f(a, b){return a.split(b)}',
+            'function f(a, b){return a["split"](b)}',
+            'function f(a, b){let x = ["split"]; return a[x[0]](b)}',
             'function f(a, b){return String.prototype.split.call(a, b)}',
             'function f(a, b){return String.prototype.split.apply(a, [b])}',
         ]
@@ -602,6 +605,9 @@ class TestJSInterpreter(unittest.TestCase):
         self._test('function f(){return "012345678".slice(-1, 1)}', '')
         self._test('function f(){return "012345678".slice(-3, -1)}', '67')
 
+    def test_splice(self):
+        self._test('function f(){var T = ["0", "1", "2"]; T["splice"](2, 1, "0")[0]; return T }', ['0', '1', '0'])
+
     def test_pop(self):
         # pop
         self._test('function f(){var a = [0, 1, 2, 3, 4, 5, 6, 7, 8]; return [a.pop(), a]}',
@@ -636,6 +642,16 @@ class TestJSInterpreter(unittest.TestCase):
                    'return [ret.length, ret[0][0], ret[1][1], ret[0][2]]}',
                    [2, 4, 1, [4, 2]])
 
+    def test_extract_function(self):
+        jsi = JSInterpreter('function a(b) { return b + 1; }')
+        func = jsi.extract_function('a')
+        self.assertEqual(func([2]), 3)
+
+    def test_extract_function_with_global_stack(self):
+        jsi = JSInterpreter('function c(d) { return d + e + f + g; }')
+        func = jsi.extract_function('c', {'e': 10}, {'f': 100, 'g': 1000})
+        self.assertEqual(func([1]), 1111)
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index 8fdcce8d4..a2d3a41ae 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -94,11 +94,51 @@ _SIG_TESTS = [
         '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
         '0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpz2ICs6EVdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
     ),
+    (
+        'https://www.youtube.com/s/player/363db69b/player_ias_tce.vflset/en_US/base.js',
+        '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+        '0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpz2ICs6EVdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+    ),
     (
         'https://www.youtube.com/s/player/4fcd6e4a/player_ias.vflset/en_US/base.js',
         '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
         'wAOAOq0QJ8ARAIgXmPlOPSBkkUs1bYFYlJCfe29xx8q7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0',
     ),
+    (
+        'https://www.youtube.com/s/player/4fcd6e4a/player_ias_tce.vflset/en_US/base.js',
+        '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+        'wAOAOq0QJ8ARAIgXmPlOPSBkkUs1bYFYlJCfe29xx8q7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0',
+    ),
+    (
+        'https://www.youtube.com/s/player/20830619/player_ias.vflset/en_US/base.js',
+        '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+        '7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw',
+    ),
+    (
+        'https://www.youtube.com/s/player/20830619/player_ias_tce.vflset/en_US/base.js',
+        '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+        '7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw',
+    ),
+    (
+        'https://www.youtube.com/s/player/20830619/player-plasma-ias-phone-en_US.vflset/base.js',
+        '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+        '7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw',
+    ),
+    (
+        'https://www.youtube.com/s/player/20830619/player-plasma-ias-tablet-en_US.vflset/base.js',
+        '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+        '7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw',
+    ),
+    (
+        'https://www.youtube.com/s/player/8a8ac953/player_ias_tce.vflset/en_US/base.js',
+        '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+        'IAOAOq0QJ8wRAAgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_E2u-m37KtXJoOySqa0',
+    ),
+    (
+        'https://www.youtube.com/s/player/8a8ac953/tv-player-es6.vflset/tv-player-es6.js',
+        '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA',
+        'IAOAOq0QJ8wRAAgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_E2u-m37KtXJoOySqa0',
+    ),
 ]
 
 _NSIG_TESTS = [
@@ -272,7 +312,7 @@ _NSIG_TESTS = [
     ),
     (
         'https://www.youtube.com/s/player/643afba4/player_ias.vflset/en_US/base.js',
-        'W9HJZKktxuYoDTqW', 'larxUlagTRAcSw',
+        'ir9-V6cdbCiyKxhr', '2PL7ZDYAALMfmA',
     ),
     (
         'https://www.youtube.com/s/player/363db69b/player_ias.vflset/en_US/base.js',
@@ -286,6 +326,26 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/4fcd6e4a/tv-player-ias.vflset/tv-player-ias.js',
         'o_L251jm8yhZkWtBW', 'lXoxI3XvToqn6A',
     ),
+    (
+        'https://www.youtube.com/s/player/20830619/tv-player-ias.vflset/tv-player-ias.js',
+        'ir9-V6cdbCiyKxhr', '9YE85kNjZiS4',
+    ),
+    (
+        'https://www.youtube.com/s/player/20830619/player-plasma-ias-phone-en_US.vflset/base.js',
+        'ir9-V6cdbCiyKxhr', '9YE85kNjZiS4',
+    ),
+    (
+        'https://www.youtube.com/s/player/20830619/player-plasma-ias-tablet-en_US.vflset/base.js',
+        'ir9-V6cdbCiyKxhr', '9YE85kNjZiS4',
+    ),
+    (
+        'https://www.youtube.com/s/player/8a8ac953/player_ias_tce.vflset/en_US/base.js',
+        'MiBYeXx_vRREbiCCmh', 'RtZYMVvmkE0JE',
+    ),
+    (
+        'https://www.youtube.com/s/player/8a8ac953/tv-player-es6.vflset/tv-player-es6.js',
+        'MiBYeXx_vRREbiCCmh', 'RtZYMVvmkE0JE',
+    ),
 ]
 
 
@@ -335,7 +395,7 @@ def t_factory(name, sig_func, url_pattern):
         test_id = re.sub(r'[/.-]', '_', m.group('id') or m.group('compat_id'))
 
         def test_func(self):
-            basename = 'player-{0}-{1}.js'.format(name, test_id)
+            basename = 'player-{0}.js'.format(test_id)
             fn = os.path.join(self.TESTDATA_DIR, basename)
 
             if not os.path.exists(fn):
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 7f8c0c532..54073ef86 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1652,7 +1652,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         assert os.path.basename(func_id) == func_id
 
         self.write_debug('Extracting signature function {0}'.format(func_id))
-        cache_spec, code = self.cache.load('youtube-sigfuncs', func_id), None
+        cache_spec, code = self.cache.load('youtube-sigfuncs', func_id, min_ver='2025.04.07'), None
 
         if not cache_spec:
             code = self._load_player(video_id, player_url, player_id)
@@ -1813,6 +1813,10 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         return ret
 
     def _extract_n_function_name(self, jscode):
+        func_name, idx = None, None
+        # these special cases are redundant and probably obsolete (2025-04):
+        # they make the tests run ~10% faster without fallback warnings
+        r"""
         func_name, idx = self._search_regex(
             # (y=NuD(),Mw(k),q=k.Z[y]||null)&&(q=narray[idx](q),k.set(y,q),k.V||NuD(''))}};
             # (R="nn"[+J.Z],mW(J),N=J.K[R]||null)&&(N=narray[idx](N),J.set(R,N))}};
@@ -1839,9 +1843,28 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                     \(\s*[\w$]+\s*\)
             ''', jscode, 'Initial JS player n function name', group=('nfunc', 'idx'),
             default=(None, None))
+        """
+
+        if not func_name:
+            # nfunc=function(x){...}|function nfunc(x); ...
+            # ... var y=[nfunc]|y[idx]=nfunc);
+            # obvious REs hang, so use a two-stage tactic
+            for m in re.finditer(r'''(?x)
+                    [\n;]var\s(?:(?:(?!,).)+,|\s)*?(?!\d)[\w$]+(?:\[(?P<idx>\d+)\])?\s*=\s*
+                        (?(idx)|\[\s*)(?P<nfunc>(?!\d)[\w$]+)(?(idx)|\s*\])
+                    \s*?[;\n]
+                    ''', jscode):
+                func_name = self._search_regex(
+                    r'[;,]\s*(function\s+)?({0})(?(1)|\s*=\s*function)\s*\((?!\d)[\w$]+\)\s*\{1}(?!\s*return\s)'.format(
+                        re.escape(m.group('nfunc')), '{'),
+                    jscode, 'Initial JS player n function name (2)', group=2, default=None)
+                if func_name:
+                    idx = m.group('idx')
+                    break
+
         # thx bashonly: yt-dlp/yt-dlp/pull/10611
         if not func_name:
-            self.report_warning('Falling back to generic n function search')
+            self.report_warning('Falling back to generic n function search', only_once=True)
             return self._search_regex(
                 r'''(?xs)
                     (?:(?<=[^\w$])|^)       # instead of \b, which ignores $
@@ -1855,14 +1878,14 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             return func_name
 
         return self._search_json(
-            r'var\s+{0}\s*='.format(re.escape(func_name)), jscode,
+            r'(?<![\w-])var\s(?:(?:(?!,).)+,|\s)*?{0}\s*='.format(re.escape(func_name)), jscode,
             'Initial JS player n function list ({0}.{1})'.format(func_name, idx),
-            func_name, contains_pattern=r'\[[\s\S]+\]', end_pattern='[,;]',
+            func_name, contains_pattern=r'\[.+\]', end_pattern='[,;]',
             transform_source=js_to_json)[int(idx)]
 
     def _extract_n_function_code(self, video_id, player_url):
         player_id = self._extract_player_info(player_url)
-        func_code = self.cache.load('youtube-nsig', player_id)
+        func_code = self.cache.load('youtube-nsig', player_id, min_ver='2025.04.07')
         jscode = func_code or self._load_player(video_id, player_url)
         jsi = JSInterpreter(jscode)
 
diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py
index ff9894f52..7630e2099 100644
--- a/youtube_dl/jsinterp.py
+++ b/youtube_dl/jsinterp.py
@@ -303,8 +303,6 @@ _UNARY_OPERATORS_X = (
     ('!', _js_unary_op(lambda x: _js_ternary(x, if_true=False, if_false=True))),
 )
 
-_OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS))
-
 _COMP_OPERATORS = (
     ('===', _js_id_op(operator.is_)),
     ('!==', _js_id_op(operator.is_not)),
@@ -316,9 +314,12 @@ _COMP_OPERATORS = (
     ('>', _js_comp_op(operator.gt)),
 )
 
+_OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS + _SC_OPERATORS))
+
 _NAME_RE = r'[a-zA-Z_$][\w$]*'
 _MATCHING_PARENS = dict(zip(*zip('()', '{}', '[]')))
 _QUOTES = '\'"/'
+_NESTED_BRACKETS = r'[^[\]]+(?:\[[^[\]]+(?:\[[^\]]+\])?\])?'
 
 
 class JS_Break(ExtractorError):
@@ -1088,15 +1089,18 @@ class JSInterpreter(object):
 
         m = re.match(r'''(?x)
             (?P<assign>
-                (?P<out>{_NAME_RE})(?:\[(?P<out_idx>(?:.+?\]\s*\[)*.+?)\])?\s*
+                (?P<out>{_NAME_RE})(?P<out_idx>(?:\[{_NESTED_BRACKETS}\])+)?\s*
                 (?P<op>{_OPERATOR_RE})?
                 =(?!=)(?P<expr>.*)$
             )|(?P<return>
                 (?!if|return|true|false|null|undefined|NaN|Infinity)(?P<name>{_NAME_RE})$
-            )|(?P<indexing>
-                (?P<in>{_NAME_RE})\[(?P<in_idx>(?:.+?\]\s*\[)*.+?)\]$
             )|(?P<attribute>
-                (?P<var>{_NAME_RE})(?:(?P<nullish>\?)?\.(?P<member>[^(]+)|\[(?P<member2>[^\]]+)\])\s*
+                (?P<var>{_NAME_RE})(?:
+                    (?P<nullish>\?)?\.(?P<member>[^(]+)|
+                    \[(?P<member2>{_NESTED_BRACKETS})\]
+                )\s*
+            )|(?P<indexing>
+                (?P<in>{_NAME_RE})(?P<in_idx>\[.+\])$
             )|(?P<function>
                 (?P<fname>{_NAME_RE})\((?P<args>.*)\)$
             )'''.format(**globals()), expr)
@@ -1111,10 +1115,11 @@ class JSInterpreter(object):
             elif left_val in (None, JS_Undefined):
                 raise self.Exception('Cannot index undefined variable ' + m.group('out'), expr=expr)
 
-            indexes = re.split(r'\]\s*\[', m.group('out_idx'))
-            for i, idx in enumerate(indexes, 1):
+            indexes = md['out_idx']
+            while indexes:
+                idx, indexes = self._separate_at_paren(indexes)
                 idx = self.interpret_expression(idx, local_vars, allow_recursion)
-                if i < len(indexes):
+                if indexes:
                     left_val = self._index(left_val, idx)
             if isinstance(idx, float):
                 idx = int(idx)
@@ -1159,7 +1164,9 @@ class JSInterpreter(object):
 
         if md.get('indexing'):
             val = local_vars[m.group('in')]
-            for idx in re.split(r'\]\s*\[', m.group('in_idx')):
+            indexes = m.group('in_idx')
+            while indexes:
+                idx, indexes = self._separate_at_paren(indexes)
                 idx = self.interpret_expression(idx, local_vars, allow_recursion)
                 val = self._index(val, idx)
             return val, should_return
@@ -1204,7 +1211,7 @@ class JSInterpreter(object):
                 if obj is JS_Undefined:
                     try:
                         if variable not in self._objects:
-                            self._objects[variable] = self.extract_object(variable)
+                            self._objects[variable] = self.extract_object(variable, local_vars)
                         obj = self._objects[variable]
                     except self.Exception:
                         if not nullish:
@@ -1215,7 +1222,7 @@ class JSInterpreter(object):
 
                 # Member access
                 if arg_str is None:
-                    return self._index(obj, member)
+                    return self._index(obj, member, nullish)
 
                 # Function call
                 argvals = [
@@ -1400,7 +1407,7 @@ class JSInterpreter(object):
         for v in self._separate(list_txt):
             yield self.interpret_expression(v, local_vars, allow_recursion)
 
-    def extract_object(self, objname):
+    def extract_object(self, objname, *global_stack):
         _FUNC_NAME_RE = r'''(?:{n}|"{n}"|'{n}')'''.format(n=_NAME_RE)
         obj = {}
         fields = next(filter(None, (
@@ -1421,7 +1428,8 @@ class JSInterpreter(object):
                 fields):
             argnames = self.build_arglist(f.group('args'))
             name = remove_quotes(f.group('key'))
-            obj[name] = function_with_repr(self.build_function(argnames, f.group('code')), 'F<{0}>'.format(name))
+            obj[name] = function_with_repr(
+                self.build_function(argnames, f.group('code'), *global_stack), 'F<{0}>'.format(name))
 
         return obj
 
diff --git a/youtube_dl/version.py b/youtube_dl/version.py
index b82fbc702..c70d9d2af 100644
--- a/youtube_dl/version.py
+++ b/youtube_dl/version.py
@@ -1,3 +1,3 @@
 from __future__ import unicode_literals
 
-__version__ = '2021.12.17'
+__version__ = '2025.04.07'

From ec75141bf03b66ffc183992ec6bb50c4ff392dd8 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 2 May 2025 13:17:18 +0100
Subject: [PATCH 855/860] [Cache] Add `clear` function

---
 test/test_cache.py  | 12 ++++++++++++
 youtube_dl/cache.py | 42 +++++++++++++++++++++++++++++++-----------
 2 files changed, 43 insertions(+), 11 deletions(-)

diff --git a/test/test_cache.py b/test/test_cache.py
index 21dfb6cfc..0431f4f15 100644
--- a/test/test_cache.py
+++ b/test/test_cache.py
@@ -66,6 +66,18 @@ class TestCache(unittest.TestCase):
         new_version = '.'.join(('%0.2d' % ((v + 1) if i == 0 else v, )) for i, v in enumerate(version_tuple(__version__)))
         self.assertIs(c.load('test_cache', 'k.', min_ver=new_version), None)
 
+    def test_cache_clear(self):
+        ydl = FakeYDL({
+            'cachedir': self.test_dir,
+        })
+        c = Cache(ydl)
+        c.store('test_cache', 'k.', 'kay')
+        c.store('test_cache', 'l.', 'ell')
+        self.assertEqual(c.load('test_cache', 'k.'), 'kay')
+        c.clear('test_cache', 'k.')
+        self.assertEqual(c.load('test_cache', 'k.'), None)
+        self.assertEqual(c.load('test_cache', 'l.'), 'ell')
+
 
 if __name__ == '__main__':
     unittest.main()
diff --git a/youtube_dl/cache.py b/youtube_dl/cache.py
index ba12d0373..eb0a729c2 100644
--- a/youtube_dl/cache.py
+++ b/youtube_dl/cache.py
@@ -1,6 +1,7 @@
 # coding: utf-8
 from __future__ import unicode_literals
 
+import errno
 import json
 import os
 import re
@@ -8,7 +9,6 @@ import shutil
 import traceback
 
 from .compat import (
-    compat_contextlib_suppress,
     compat_getenv,
     compat_open as open,
     compat_os_makedirs,
@@ -78,6 +78,22 @@ class Cache(object):
             tb = traceback.format_exc()
             self._report_warning('Writing cache to {fn!r} failed: {tb}'.format(fn=fn, tb=tb))
 
+    def clear(self, section, key, dtype='json'):
+
+        if not self.enabled:
+            return
+
+        fn = self._get_cache_fn(section, key, dtype)
+        self._write_debug('Clearing {section}.{key} from cache'.format(section=section, key=key))
+        try:
+            os.remove(fn)
+        except Exception as e:
+            if getattr(e, 'errno') == errno.ENOENT:
+                # file not found
+                return
+            tb = traceback.format_exc()
+            self._report_warning('Clearing cache from {fn!r} failed: {tb}'.format(fn=fn, tb=tb))
+
     def _validate(self, data, min_ver):
         version = traverse_obj(data, self._VERSION_KEY)
         if not version:  # Backward compatibility
@@ -94,17 +110,21 @@ class Cache(object):
             return default
 
         cache_fn = self._get_cache_fn(section, key, dtype)
-        with compat_contextlib_suppress(IOError):  # If no cache available
+        try:
+            with open(cache_fn, encoding='utf-8') as cachef:
+                self._write_debug('Loading {section}.{key} from cache'.format(section=section, key=key), only_once=True)
+                return self._validate(json.load(cachef), min_ver)
+        except (ValueError, KeyError):
             try:
-                with open(cache_fn, encoding='utf-8') as cachef:
-                    self._write_debug('Loading {section}.{key} from cache'.format(section=section, key=key), only_once=True)
-                    return self._validate(json.load(cachef), min_ver)
-            except (ValueError, KeyError):
-                try:
-                    file_size = os.path.getsize(cache_fn)
-                except (OSError, IOError) as oe:
-                    file_size = error_to_compat_str(oe)
-                self._report_warning('Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
+                file_size = 'size: %d' % os.path.getsize(cache_fn)
+            except (OSError, IOError) as oe:
+                file_size = error_to_compat_str(oe)
+            self._report_warning('Cache retrieval from %s failed (%s)' % (cache_fn, file_size))
+        except Exception as e:
+            if getattr(e, 'errno') == errno.ENOENT:
+                # no cache available
+                return
+            self._report_warning('Cache retrieval from %s failed' % (cache_fn,))
 
         return default
 

From 3a42f6ad372935733150159b0447e790c037c1af Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 2 May 2025 13:31:45 +0100
Subject: [PATCH 856/860] [YouTube] Cache signature timestamp from player JS

* if the YT webpage can't be loaded, getting the `sts` requires loading the
player JS: this caches it
* based on yt-dlp/yt-dlp#13047, thx bashonly
---
 youtube_dl/extractor/youtube.py | 142 +++++++++++++++++++++++---------
 1 file changed, 104 insertions(+), 38 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 54073ef86..a1c9a6eae 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -49,6 +49,7 @@ from ..utils import (
     parse_duration,
     parse_qs,
     qualities,
+    remove_end,
     remove_start,
     smuggle_url,
     str_or_none,
@@ -1584,6 +1585,15 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         '397': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'},
     }
 
+    _PLAYER_JS_VARIANT_MAP = (
+        ('main', 'player_ias.vflset/en_US/base.js'),
+        ('tce', 'player_ias_tce.vflset/en_US/base.js'),
+        ('tv', 'tv-player-ias.vflset/tv-player-ias.js'),
+        ('tv_es6', 'tv-player-es6.vflset/tv-player-es6.js'),
+        ('phone', 'player-plasma-ias-phone-en_US.vflset/base.js'),
+        ('tablet', 'player-plasma-ias-tablet-en_US.vflset/base.js'),
+    )
+
     @classmethod
     def suitable(cls, url):
         if parse_qs(url).get('list', [None])[0]:
@@ -1631,36 +1641,83 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             raise ExtractorError(
                 'Cannot identify player %r' % (player_url,), cause=e)
 
-    def _load_player(self, video_id, player_url, fatal=True, player_id=None):
-        if not player_id:
+    def _player_js_cache_key(self, player_url, extra_id=None, _cache={}):
+        if player_url not in _cache:
             player_id = self._extract_player_info(player_url)
-        if player_id not in self._code_cache:
+            player_path = remove_start(
+                compat_urllib_parse.urlparse(player_url).path,
+                '/s/player/{0}/'.format(player_id))
+            variant = next((k for k, v in self._PLAYER_JS_VARIANT_MAP
+                           if v == player_path), None)
+            if not variant:
+                variant = next(
+                    (k for k, v in self._PLAYER_JS_VARIANT_MAP
+                     if re.match(re.escape(v).replace('en_US', r'\w+') + '$', player_path)),
+                    None)
+            if not variant:
+                self.write_debug(
+                    'Unable to determine player JS variant\n'
+                    '        player = {0}'.format(player_url), only_once=True)
+                variant = re.sub(r'[^a-zA-Z0-9]', '_', remove_end(player_path, '.js'))
+            _cache[player_url] = join_nonempty(player_id, variant)
+
+        if extra_id:
+            extra_id = '-'.join((_cache[player_url], extra_id))
+            assert os.path.basename(extra_id) == extra_id
+            return extra_id
+        return _cache[player_url]
+
+    def _load_player(self, video_id, player_url, fatal=True):
+        player_js_key = self._player_js_cache_key(player_url)
+        if player_js_key not in self._code_cache:
             code = self._download_webpage(
                 player_url, video_id, fatal=fatal,
-                note='Downloading player ' + player_id,
-                errnote='Download of %s failed' % player_url)
+                note='Downloading player {0}'.format(player_js_key),
+                errnote='Download of {0} failed'.format(player_url))
             if code:
-                self._code_cache[player_id] = code
-        return self._code_cache[player_id] if fatal else self._code_cache.get(player_id)
+                self._code_cache[player_js_key] = code
+        return self._code_cache.get(player_js_key)
+
+    def _load_player_data_from_cache(self, name, player_url, extra_id=None):
+        cache_id = ('youtube-{0}'.format(name), self._player_js_cache_key(player_url, extra_id))
+        data = self._player_cache.get(cache_id)
+        if data:
+            return data
+
+        data = self.cache.load(*cache_id, min_ver='2025.04.07')
+        if data:
+            self._player_cache[cache_id] = data
+        return data
+
+    def _store_player_data_to_cache(self, name, player_url, data, extra_id=None):
+        cache_id = ('youtube-{0}'.format(name), self._player_js_cache_key(player_url, extra_id))
+
+        if cache_id not in self._player_cache:
+            self.cache.store(cache_id[0], cache_id[1], data)
+            self._player_cache[cache_id] = data
 
     def _extract_signature_function(self, video_id, player_url, example_sig):
-        player_id = self._extract_player_info(player_url)
+        # player_id = self._extract_player_info(player_url)
 
         # Read from filesystem cache
-        func_id = 'js_{0}_{1}'.format(
-            player_id, self._signature_cache_id(example_sig))
-        assert os.path.basename(func_id) == func_id
-
-        self.write_debug('Extracting signature function {0}'.format(func_id))
-        cache_spec, code = self.cache.load('youtube-sigfuncs', func_id, min_ver='2025.04.07'), None
+        extra_id = self._signature_cache_id(example_sig)
+        self.write_debug('Extracting signature function {0}-{1}'.format(player_url, extra_id))
+        cache_spec, code = self._load_player_data_from_cache(
+            'sigfuncs', player_url, extra_id=extra_id, min_ver='2025.04.07'
+        ), None
 
         if not cache_spec:
-            code = self._load_player(video_id, player_url, player_id)
-        if code:
-            res = self._parse_sig_js(code)
-            test_string = ''.join(map(compat_chr, range(len(example_sig))))
-            cache_spec = [ord(c) for c in res(test_string)]
-            self.cache.store('youtube-sigfuncs', func_id, cache_spec)
+            code = self._load_player(video_id, player_url)
+            if code:
+                res = self._parse_sig_js(code)
+                test_string = ''.join(map(compat_chr, range(len(example_sig))))
+                cache_spec = [ord(c) for c in res(test_string)]
+                self._store_player_data_to_cache(
+                    'sigfuncs', player_url, cache_spec, extra_id=extra_id)
+            else:
+                self.report_warning(
+                    'Failed to compute signature function {0}-{1}'.format(
+                        player_url, extra_id))
 
         return lambda s: ''.join(s[i] for i in cache_spec)
 
@@ -1885,22 +1942,21 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     def _extract_n_function_code(self, video_id, player_url):
         player_id = self._extract_player_info(player_url)
-        func_code = self.cache.load('youtube-nsig', player_id, min_ver='2025.04.07')
+        func_code = self._load_player_data_from_cache('nsig', player_url)
         jscode = func_code or self._load_player(video_id, player_url)
         jsi = JSInterpreter(jscode)
 
         if func_code:
             return jsi, player_id, func_code
-        return self._extract_n_function_code_jsi(video_id, jsi, player_id)
 
-    def _extract_n_function_code_jsi(self, video_id, jsi, player_id=None):
+        return self._extract_n_function_code_jsi(video_id, jsi, player_id, player_url)
 
+    def _extract_n_function_code_jsi(self, video_id, jsi, player_id=None, player_url=None):
         func_name = self._extract_n_function_name(jsi.code)
 
         func_code = self._extract_sig_fn(jsi, func_name)
-
-        if player_id:
-            self.cache.store('youtube-nsig', player_id, func_code)
+        if player_url:
+            self._store_player_data_to_cache('nsig', player_url, func_code)
         return jsi, player_id, func_code
 
     def _extract_n_function_from_code(self, jsi, func_code):
@@ -1944,18 +2000,28 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         Required to tell API what sig/player version is in use.
         """
         sts = traverse_obj(ytcfg, 'STS', expected_type=int)
-        if not sts:
-            # Attempt to extract from player
-            if player_url is None:
-                error_msg = 'Cannot extract signature timestamp without player_url.'
-                if fatal:
-                    raise ExtractorError(error_msg)
-                self.report_warning(error_msg)
-                return
-            code = self._load_player(video_id, player_url, fatal=fatal)
-            sts = int_or_none(self._search_regex(
-                r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code or '',
-                'JS player signature timestamp', group='sts', fatal=fatal))
+        if sts:
+            return sts
+
+        if not player_url:
+            error_msg = 'Cannot extract signature timestamp without player url'
+            if fatal:
+                raise ExtractorError(error_msg)
+            self.report_warning(error_msg)
+            return None
+
+        sts = self._load_player_data_from_cache('sts', player_url)
+        if sts:
+            return sts
+
+        # Attempt to extract from player
+        code = self._load_player(video_id, player_url, fatal=fatal)
+        sts = int_or_none(self._search_regex(
+            r'(?:signatureTimestamp|sts)\s*:\s*(?P<sts>[0-9]{5})', code or '',
+            'JS player signature timestamp', group='sts', fatal=fatal))
+        if sts:
+            self._store_player_data_to_cache('sts', player_url, sts)
+
         return sts
 
     def _mark_watched(self, video_id, player_response):

From 4a31290ae14705100814b88f158e7876f83a5b67 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 2 May 2025 13:38:32 +0100
Subject: [PATCH 857/860] [YouTube] Delete cached problem nsig cache data on
 descrambling error

* inspired by yt-dlp/yt-dlp#12750
---
 youtube_dl/extractor/youtube.py | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index a1c9a6eae..0d6ffa3f2 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1696,6 +1696,13 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             self.cache.store(cache_id[0], cache_id[1], data)
             self._player_cache[cache_id] = data
 
+    def _remove_player_data_from_cache(self, name, player_url, extra_id=None):
+        cache_id = ('youtube-{0}'.format(name), self._player_js_cache_key(player_url, extra_id))
+
+        if cache_id in self._player_cache:
+            self.cache.clear(*cache_id)
+            self._player_cache.pop(cache_id, None)
+
     def _extract_signature_function(self, video_id, player_url, example_sig):
         # player_id = self._extract_player_info(player_url)
 
@@ -1989,7 +1996,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
             n_param = n_param[-1]
             n_response = decrypt_nsig(n_param)(n_param, video_id, player_url)
             if n_response is None:
-                # give up if descrambling failed
+                # give up and forget cached data if descrambling failed
+                self._remove_player_data_from_cache('nsig', player_url)
                 break
             fmt['url'] = update_url_query(fmt['url'], {'n': n_response})
 

From 680069a14958187cf576ed91c96d2f273aea711f Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Fri, 2 May 2025 13:49:05 +0100
Subject: [PATCH 858/860] [YouTube] Improve n-sig function extraction for
 player `aa3fc80b`

Resolves #33123
---
 test/test_youtube_signature.py  |  4 ++++
 youtube_dl/extractor/youtube.py | 34 +++++++++++++++++++++------------
 2 files changed, 26 insertions(+), 12 deletions(-)

diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py
index a2d3a41ae..98221b9c2 100644
--- a/test/test_youtube_signature.py
+++ b/test/test_youtube_signature.py
@@ -346,6 +346,10 @@ _NSIG_TESTS = [
         'https://www.youtube.com/s/player/8a8ac953/tv-player-es6.vflset/tv-player-es6.js',
         'MiBYeXx_vRREbiCCmh', 'RtZYMVvmkE0JE',
     ),
+    (
+        'https://www.youtube.com/s/player/aa3fc80b/player_ias.vflset/en_US/base.js',
+        '0qY9dal2uzOnOGwa-48hha', 'VSh1KDfQMk-eag',
+    ),
 ]
 
 
diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 0d6ffa3f2..7290ae813 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1878,6 +1878,20 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
 
     def _extract_n_function_name(self, jscode):
         func_name, idx = None, None
+
+        def generic_n_function_search(func_name=None):
+            return self._search_regex(
+                r'''(?xs)
+                    (?:(?<=[^\w$])|^)       # instead of \b, which ignores $
+                    (?P<name>%s)\s*=\s*function\((?!\d)[a-zA-Z\d_$]+\)
+                    \s*\{(?:(?!};).)+?(?:
+                        ["']enhanced_except_ |
+                        return\s*(?P<q>"|')[a-zA-Z\d-]+_w8_(?P=q)\s*\+\s*[\w$]+
+                    )
+                ''' % (func_name or r'(?!\d)[a-zA-Z\d_$]+',), jscode,
+                'Initial JS player n function name', group='name',
+                default=None if func_name else NO_DEFAULT)
+
         # these special cases are redundant and probably obsolete (2025-04):
         # they make the tests run ~10% faster without fallback warnings
         r"""
@@ -1918,26 +1932,22 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
                         (?(idx)|\[\s*)(?P<nfunc>(?!\d)[\w$]+)(?(idx)|\s*\])
                     \s*?[;\n]
                     ''', jscode):
-                func_name = self._search_regex(
+                fn = self._search_regex(
                     r'[;,]\s*(function\s+)?({0})(?(1)|\s*=\s*function)\s*\((?!\d)[\w$]+\)\s*\{1}(?!\s*return\s)'.format(
                         re.escape(m.group('nfunc')), '{'),
                     jscode, 'Initial JS player n function name (2)', group=2, default=None)
-                if func_name:
+                if fn:
+                    func_name = fn
                     idx = m.group('idx')
-                    break
+                    if generic_n_function_search(func_name):
+                        # don't look any further
+                        break
 
         # thx bashonly: yt-dlp/yt-dlp/pull/10611
         if not func_name:
             self.report_warning('Falling back to generic n function search', only_once=True)
-            return self._search_regex(
-                r'''(?xs)
-                    (?:(?<=[^\w$])|^)       # instead of \b, which ignores $
-                    (?P<name>(?!\d)[a-zA-Z\d_$]+)\s*=\s*function\((?!\d)[a-zA-Z\d_$]+\)
-                    \s*\{(?:(?!};).)+?(?:
-                        ["']enhanced_except_ |
-                        return\s*(?P<q>"|')[a-zA-Z\d-]+_w8_(?P=q)\s*\+\s*[\w$]+
-                    )
-                ''', jscode, 'Initial JS player n function name', group='name')
+            return generic_n_function_search()
+
         if not idx:
             return func_name
 

From e102b9993af6defb4e03699840b01c63e5623276 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sat, 3 May 2025 18:33:39 +0100
Subject: [PATCH 859/860] [workflows/ci.yml] Move pinned Ubuntu runner images
 from withdrawn 20.4 to 22.04

* fix consequent missing `python-is-python2` package
---
 .github/workflows/ci.yml | 13 ++++++++-----
 1 file changed, 8 insertions(+), 5 deletions(-)

diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index d3b9ae016..8234e0ccb 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -116,7 +116,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: [ubuntu-20.04]
+        os: [ubuntu-22.04]
         python-version: ${{ fromJSON(needs.select.outputs.cpython-versions) }}
         python-impl: [cpython]
         ytdl-test-set: ${{ fromJSON(needs.select.outputs.test-set) }}
@@ -133,12 +133,12 @@ jobs:
           ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download'  || 'nodownload' }}
           run-tests-ext: bat
         # jython
-        - os: ubuntu-20.04
+        - os: ubuntu-22.04
           python-version: 2.7
           python-impl: jython
           ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }}
           run-tests-ext: sh
-        - os: ubuntu-20.04
+        - os: ubuntu-22.04
           python-version: 2.7
           python-impl: jython
           ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download'  || 'nodownload' }}
@@ -160,7 +160,7 @@ jobs:
       # NB may run apt-get install in Linux
       uses: ytdl-org/setup-python@v1
       env:
-        # Temporary workaround for Python 3.5 failures - May 2024
+        # Temporary (?) workaround for Python 3.5 failures - May 2024
         PIP_TRUSTED_HOST: "pypi.python.org pypi.org files.pythonhosted.org"
       with:
         python-version: ${{ matrix.python-version }}
@@ -240,7 +240,10 @@ jobs:
       # install 2.7
       shell: bash
       run: |
-        sudo apt-get install -y python2 python-is-python2
+        # Ubuntu 22.04 no longer has python-is-python2: fetch it
+        curl -L "http://launchpadlibrarian.net/474693132/python-is-python2_2.7.17-4_all.deb" -o python-is-python2.deb
+        sudo apt-get install -y python2
+        sudo dpkg --force-breaks -i python-is-python2.deb
         echo "PYTHONHOME=/usr" >> "$GITHUB_ENV"
     #-------- Python 2.6 --
     - name: Set up Python 2.6 environment

From a084c80f7bac9ae343075a97cc0fb2c1c96ade89 Mon Sep 17 00:00:00 2001
From: dirkf <fieldhouse@gmx.net>
Date: Sun, 4 May 2025 12:51:54 +0100
Subject: [PATCH 860/860] [YouTube] Fix 680069a, excess `min_ver`

Resolves #33125.
---
 youtube_dl/extractor/youtube.py | 3 +--
 1 file changed, 1 insertion(+), 2 deletions(-)

diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py
index 7290ae813..b31798729 100644
--- a/youtube_dl/extractor/youtube.py
+++ b/youtube_dl/extractor/youtube.py
@@ -1710,8 +1710,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor):
         extra_id = self._signature_cache_id(example_sig)
         self.write_debug('Extracting signature function {0}-{1}'.format(player_url, extra_id))
         cache_spec, code = self._load_player_data_from_cache(
-            'sigfuncs', player_url, extra_id=extra_id, min_ver='2025.04.07'
-        ), None
+            'sigfuncs', player_url, extra_id=extra_id), None
 
         if not cache_spec:
             code = self._load_player(video_id, player_url)