diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8234e0ccb..d3b9ae016 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -116,7 +116,7 @@ jobs: strategy: fail-fast: true matrix: - os: [ubuntu-22.04] + os: [ubuntu-20.04] python-version: ${{ fromJSON(needs.select.outputs.cpython-versions) }} python-impl: [cpython] ytdl-test-set: ${{ fromJSON(needs.select.outputs.test-set) }} @@ -133,12 +133,12 @@ jobs: ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download' || 'nodownload' }} run-tests-ext: bat # jython - - os: ubuntu-22.04 + - os: ubuntu-20.04 python-version: 2.7 python-impl: jython ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }} run-tests-ext: sh - - os: ubuntu-22.04 + - os: ubuntu-20.04 python-version: 2.7 python-impl: jython ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download' || 'nodownload' }} @@ -160,7 +160,7 @@ jobs: # NB may run apt-get install in Linux uses: ytdl-org/setup-python@v1 env: - # Temporary (?) workaround for Python 3.5 failures - May 2024 + # Temporary workaround for Python 3.5 failures - May 2024 PIP_TRUSTED_HOST: "pypi.python.org pypi.org files.pythonhosted.org" with: python-version: ${{ matrix.python-version }} @@ -240,10 +240,7 @@ jobs: # install 2.7 shell: bash run: | - # Ubuntu 22.04 no longer has python-is-python2: fetch it - curl -L "http://launchpadlibrarian.net/474693132/python-is-python2_2.7.17-4_all.deb" -o python-is-python2.deb - sudo apt-get install -y python2 - sudo dpkg --force-breaks -i python-is-python2.deb + sudo apt-get install -y python2 python-is-python2 echo "PYTHONHOME=/usr" >> "$GITHUB_ENV" #-------- Python 2.6 -- - name: Set up Python 2.6 environment diff --git a/test/test_cache.py b/test/test_cache.py index 0431f4f15..931074aa1 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -63,21 +63,9 @@ class TestCache(unittest.TestCase): obj = {'x': 1, 'y': ['รค', '\\a', True]} c.store('test_cache', 'k.', obj) self.assertEqual(c.load('test_cache', 'k.', min_ver='1970.01.01'), obj) - new_version = '.'.join(('%0.2d' % ((v + 1) if i == 0 else v, )) for i, v in enumerate(version_tuple(__version__))) + new_version = '.'.join(('%d' % ((v + 1) if i == 0 else v, )) for i, v in enumerate(version_tuple(__version__))) self.assertIs(c.load('test_cache', 'k.', min_ver=new_version), None) - def test_cache_clear(self): - ydl = FakeYDL({ - 'cachedir': self.test_dir, - }) - c = Cache(ydl) - c.store('test_cache', 'k.', 'kay') - c.store('test_cache', 'l.', 'ell') - self.assertEqual(c.load('test_cache', 'k.'), 'kay') - c.clear('test_cache', 'k.') - self.assertEqual(c.load('test_cache', 'k.'), None) - self.assertEqual(c.load('test_cache', 'l.'), 'ell') - if __name__ == '__main__': unittest.main() diff --git a/test/test_jsinterp.py b/test/test_jsinterp.py index 479cb43a0..104e766be 100644 --- a/test/test_jsinterp.py +++ b/test/test_jsinterp.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 from __future__ import unicode_literals @@ -7,14 +6,12 @@ from __future__ import unicode_literals import os import sys import unittest - sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import math import re -import time -from youtube_dl.compat import compat_str as str +from youtube_dl.compat import compat_str from youtube_dl.jsinterp import JS_Undefined, JSInterpreter NaN = object() @@ -22,7 +19,7 @@ NaN = object() class TestJSInterpreter(unittest.TestCase): def _test(self, jsi_or_code, expected, func='f', args=()): - if isinstance(jsi_or_code, str): + if isinstance(jsi_or_code, compat_str): jsi_or_code = JSInterpreter(jsi_or_code) got = jsi_or_code.call_function(func, *args) if expected is NaN: @@ -43,27 +40,16 @@ class TestJSInterpreter(unittest.TestCase): self._test('function f(){return 42 + 7;}', 49) self._test('function f(){return 42 + undefined;}', NaN) self._test('function f(){return 42 + null;}', 42) - self._test('function f(){return 1 + "";}', '1') - self._test('function f(){return 42 + "7";}', '427') - self._test('function f(){return false + true;}', 1) - self._test('function f(){return "false" + true;}', 'falsetrue') - self._test('function f(){return ' - '1 + "2" + [3,4] + {k: 56} + null + undefined + Infinity;}', - '123,4[object Object]nullundefinedInfinity') def test_sub(self): self._test('function f(){return 42 - 7;}', 35) self._test('function f(){return 42 - undefined;}', NaN) self._test('function f(){return 42 - null;}', 42) - self._test('function f(){return 42 - "7";}', 35) - self._test('function f(){return 42 - "spam";}', NaN) def test_mul(self): self._test('function f(){return 42 * 7;}', 294) self._test('function f(){return 42 * undefined;}', NaN) self._test('function f(){return 42 * null;}', 0) - self._test('function f(){return 42 * "7";}', 294) - self._test('function f(){return 42 * "eggs";}', NaN) def test_div(self): jsi = JSInterpreter('function f(a, b){return a / b;}') @@ -71,26 +57,17 @@ class TestJSInterpreter(unittest.TestCase): self._test(jsi, NaN, args=(JS_Undefined, 1)) self._test(jsi, float('inf'), args=(2, 0)) self._test(jsi, 0, args=(0, 3)) - self._test(jsi, 6, args=(42, 7)) - self._test(jsi, 0, args=(42, float('inf'))) - self._test(jsi, 6, args=("42", 7)) - self._test(jsi, NaN, args=("spam", 7)) def test_mod(self): self._test('function f(){return 42 % 7;}', 0) self._test('function f(){return 42 % 0;}', NaN) self._test('function f(){return 42 % undefined;}', NaN) - self._test('function f(){return 42 % "7";}', 0) - self._test('function f(){return 42 % "beans";}', NaN) def test_exp(self): self._test('function f(){return 42 ** 2;}', 1764) self._test('function f(){return 42 ** undefined;}', NaN) self._test('function f(){return 42 ** null;}', 1) - self._test('function f(){return undefined ** 0;}', 1) self._test('function f(){return undefined ** 42;}', NaN) - self._test('function f(){return 42 ** "2";}', 1764) - self._test('function f(){return 42 ** "spam";}', NaN) def test_calc(self): self._test('function f(a){return 2*a+1;}', 7, args=[3]) @@ -112,60 +89,13 @@ class TestJSInterpreter(unittest.TestCase): self._test('function f(){return 19 & 21;}', 17) self._test('function f(){return 11 >> 2;}', 2) self._test('function f(){return []? 2+3: 4;}', 5) - # equality - self._test('function f(){return 1 == 1}', True) - self._test('function f(){return 1 == 1.0}', True) - self._test('function f(){return 1 == "1"}', True) self._test('function f(){return 1 == 2}', False) - self._test('function f(){return 1 != "1"}', False) - self._test('function f(){return 1 != 2}', True) - self._test('function f(){var x = {a: 1}; var y = x; return x == y}', True) - self._test('function f(){var x = {a: 1}; return x == {a: 1}}', False) - self._test('function f(){return NaN == NaN}', False) - self._test('function f(){return null == undefined}', True) - self._test('function f(){return "spam, eggs" == "spam, eggs"}', True) - # strict equality - self._test('function f(){return 1 === 1}', True) - self._test('function f(){return 1 === 1.0}', True) - self._test('function f(){return 1 === "1"}', False) - self._test('function f(){return 1 === 2}', False) - self._test('function f(){var x = {a: 1}; var y = x; return x === y}', True) - self._test('function f(){var x = {a: 1}; return x === {a: 1}}', False) - self._test('function f(){return NaN === NaN}', False) - self._test('function f(){return null === undefined}', False) - self._test('function f(){return null === null}', True) - self._test('function f(){return undefined === undefined}', True) - self._test('function f(){return "uninterned" === "uninterned"}', True) - self._test('function f(){return 1 === 1}', True) - self._test('function f(){return 1 === "1"}', False) - self._test('function f(){return 1 !== 1}', False) - self._test('function f(){return 1 !== "1"}', True) - # expressions self._test('function f(){return 0 && 1 || 2;}', 2) self._test('function f(){return 0 ?? 42;}', 0) self._test('function f(){return "life, the universe and everything" < 42;}', False) # https://github.com/ytdl-org/youtube-dl/issues/32815 self._test('function f(){return 0 - 7 * - 6;}', 42) - def test_bitwise_operators_typecast(self): - # madness - self._test('function f(){return null << 5}', 0) - self._test('function f(){return undefined >> 5}', 0) - self._test('function f(){return 42 << NaN}', 42) - self._test('function f(){return 42 << Infinity}', 42) - self._test('function f(){return 0.0 << null}', 0) - self._test('function f(){return NaN << 42}', 0) - self._test('function f(){return "21.9" << 1}', 42) - self._test('function f(){return true << "5";}', 32) - self._test('function f(){return true << true;}', 2) - self._test('function f(){return "19" & "21.9";}', 17) - self._test('function f(){return "19" & false;}', 0) - self._test('function f(){return "11.0" >> "2.1";}', 2) - self._test('function f(){return 5 ^ 9;}', 12) - self._test('function f(){return 0.0 << NaN}', 0) - self._test('function f(){return null << undefined}', 0) - self._test('function f(){return 21 << 4294967297}', 42) - def test_array_access(self): self._test('function f(){var x = [1,2,3]; x[0] = 4; x[0] = 5; x[2.0] = 7; return x;}', [5, 2, 7]) @@ -180,8 +110,8 @@ class TestJSInterpreter(unittest.TestCase): self._test('function f(){var x = 20; x = 30 + 1; return x;}', 31) self._test('function f(){var x = 20; x += 30 + 1; return x;}', 51) self._test('function f(){var x = 20; x -= 30 + 1; return x;}', -11) - self._test('function f(){var x = 2; var y = ["a", "b"]; y[x%y["length"]]="z"; return y}', ['z', 'b']) + @unittest.skip('Not yet fully implemented') def test_comments(self): self._test(''' function f() { @@ -200,15 +130,6 @@ class TestJSInterpreter(unittest.TestCase): } ''', 3) - self._test(''' - function f() { - var x = ( /* 1 + */ 2 + - /* 30 * 40 */ - 50); - return x; - } - ''', 52) - def test_precedence(self): self._test(''' function f() { @@ -230,34 +151,6 @@ class TestJSInterpreter(unittest.TestCase): self._test(jsi, 86000, args=['12/31/1969 18:01:26 MDT']) # epoch 0 self._test(jsi, 0, args=['1 January 1970 00:00:00 UTC']) - # undefined - self._test(jsi, NaN, args=[JS_Undefined]) - # y,m,d, ... - may fail with older dates lacking DST data - jsi = JSInterpreter( - 'function f() { return new Date(%s); }' - % ('2024, 5, 29, 2, 52, 12, 42',)) - self._test(jsi, ( - 1719625932042 # UK value - + ( - + 3600 # back to GMT - + (time.altzone if time.daylight # host's DST - else time.timezone) - ) * 1000)) - # no arg - self.assertAlmostEqual(JSInterpreter( - 'function f() { return new Date() - 0; }').call_function('f'), - time.time() * 1000, delta=100) - # Date.now() - self.assertAlmostEqual(JSInterpreter( - 'function f() { return Date.now(); }').call_function('f'), - time.time() * 1000, delta=100) - # Date.parse() - jsi = JSInterpreter('function f(dt) { return Date.parse(dt); }') - self._test(jsi, 0, args=['1 January 1970 00:00:00 UTC']) - # Date.UTC() - jsi = JSInterpreter('function f() { return Date.UTC(%s); }' - % ('1970, 0, 1, 0, 0, 0, 0',)) - self._test(jsi, 0) def test_call(self): jsi = JSInterpreter(''' @@ -372,28 +265,8 @@ class TestJSInterpreter(unittest.TestCase): self._test('function f() { a=5; return (a -= 1, a+=3, a); }', 7) self._test('function f() { return (l=[0,1,2,3], function(a, b){return a+b})((l[1], l[2]), l[3]) }', 5) - def test_not(self): - self._test('function f() { return ! undefined; }', True) - self._test('function f() { return !0; }', True) - self._test('function f() { return !!0; }', False) - self._test('function f() { return ![]; }', False) - self._test('function f() { return !0 !== false; }', True) - def test_void(self): - self._test('function f() { return void 42; }', JS_Undefined) - - def test_typeof(self): - self._test('function f() { return typeof undefined; }', 'undefined') - self._test('function f() { return typeof NaN; }', 'number') - self._test('function f() { return typeof Infinity; }', 'number') - self._test('function f() { return typeof true; }', 'boolean') - self._test('function f() { return typeof null; }', 'object') - self._test('function f() { return typeof "a string"; }', 'string') - self._test('function f() { return typeof 42; }', 'number') - self._test('function f() { return typeof 42.42; }', 'number') - self._test('function f() { var g = function(){}; return typeof g; }', 'function') - self._test('function f() { return typeof {key: "value"}; }', 'object') - # not yet implemented: Symbol, BigInt + self._test('function f() { return void 42; }', None) def test_return_function(self): jsi = JSInterpreter(''' @@ -410,7 +283,7 @@ class TestJSInterpreter(unittest.TestCase): def test_undefined(self): self._test('function f() { return undefined === undefined; }', True) self._test('function f() { return undefined; }', JS_Undefined) - self._test('function f() { return undefined ?? 42; }', 42) + self._test('function f() {return undefined ?? 42; }', 42) self._test('function f() { let v; return v; }', JS_Undefined) self._test('function f() { let v; return v**0; }', 1) self._test('function f() { let v; return [v>42, v<=42, v&&42, 42&&v]; }', @@ -451,19 +324,8 @@ class TestJSInterpreter(unittest.TestCase): self._test('function f() { let a; return a?.qq; }', JS_Undefined) self._test('function f() { let a = {m1: 42, m2: 0 }; return a?.qq; }', JS_Undefined) - def test_indexing(self): - self._test('function f() { return [1, 2, 3, 4][3]}', 4) - self._test('function f() { return [1, [2, [3, [4]]]][1][1][1][0]}', 4) - self._test('function f() { var o = {1: 2, 3: 4}; return o[3]}', 4) - self._test('function f() { var o = {1: 2, 3: 4}; return o["3"]}', 4) - self._test('function f() { return [1, [2, {3: [4]}]][1][1]["3"][0]}', 4) - self._test('function f() { return [1, 2, 3, 4].length}', 4) - self._test('function f() { var o = {1: 2, 3: 4}; return o.length}', JS_Undefined) - self._test('function f() { var o = {1: 2, 3: 4}; o["length"] = 42; return o.length}', 42) - def test_regex(self): self._test('function f() { let a=/,,[/,913,/](,)}/; }', None) - self._test('function f() { let a=/,,[/,913,/](,)}/; return a.source; }', ',,[/,913,/](,)}') jsi = JSInterpreter(''' function x() { let a=/,,[/,913,/](,)}/; "".replace(a, ""); return a; } @@ -511,6 +373,13 @@ class TestJSInterpreter(unittest.TestCase): self._test('function f(){return -524999584 << 5}', 379882496) self._test('function f(){return 1236566549 << 5}', 915423904) + def test_bitwise_operators_typecast(self): + # madness + self._test('function f(){return null << 5}', 0) + self._test('function f(){return undefined >> 5}', 0) + self._test('function f(){return 42 << NaN}', 42) + self._test('function f(){return 42 << Infinity}', 42) + def test_negative(self): self._test('function f(){return 2 * -2.0 ;}', -4) self._test('function f(){return 2 - - -2 ;}', 0) @@ -542,19 +411,10 @@ class TestJSInterpreter(unittest.TestCase): self._test(jsi, 't-e-s-t', args=[test_input, '-']) self._test(jsi, '', args=[[], '-']) - self._test('function f(){return ' - '[1, 1.0, "abc", {a: 1}, null, undefined, Infinity, NaN].join()}', - '1,1,abc,[object Object],,,Infinity,NaN') - self._test('function f(){return ' - '[1, 1.0, "abc", {a: 1}, null, undefined, Infinity, NaN].join("~")}', - '1~1~abc~[object Object]~~~Infinity~NaN') - def test_split(self): test_result = list('test') tests = [ 'function f(a, b){return a.split(b)}', - 'function f(a, b){return a["split"](b)}', - 'function f(a, b){let x = ["split"]; return a[x[0]](b)}', 'function f(a, b){return String.prototype.split.call(a, b)}', 'function f(a, b){return String.prototype.split.apply(a, [b])}', ] @@ -564,93 +424,6 @@ class TestJSInterpreter(unittest.TestCase): self._test(jsi, test_result, args=['t-e-s-t', '-']) self._test(jsi, [''], args=['', '-']) self._test(jsi, [], args=['', '']) - # RegExp split - self._test('function f(){return "test".split(/(?:)/)}', - ['t', 'e', 's', 't']) - self._test('function f(){return "t-e-s-t".split(/[es-]+/)}', - ['t', 't']) - # from MDN: surrogate pairs aren't handled: case 1 fails - # self._test('function f(){return "๐Ÿ˜„๐Ÿ˜„".split(/(?:)/)}', - # ['\ud83d', '\ude04', '\ud83d', '\ude04']) - # case 2 beats Py3.2: it gets the case 1 result - if sys.version_info >= (2, 6) and not ((3, 0) <= sys.version_info < (3, 3)): - self._test('function f(){return "๐Ÿ˜„๐Ÿ˜„".split(/(?:)/u)}', - ['๐Ÿ˜„', '๐Ÿ˜„']) - - def test_slice(self): - self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice()}', [0, 1, 2, 3, 4, 5, 6, 7, 8]) - self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(0)}', [0, 1, 2, 3, 4, 5, 6, 7, 8]) - self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(5)}', [5, 6, 7, 8]) - self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(99)}', []) - self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(-2)}', [7, 8]) - self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(-99)}', [0, 1, 2, 3, 4, 5, 6, 7, 8]) - self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(0, 0)}', []) - self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(1, 0)}', []) - self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(0, 1)}', [0]) - self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(3, 6)}', [3, 4, 5]) - self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(1, -1)}', [1, 2, 3, 4, 5, 6, 7]) - self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(-1, 1)}', []) - self._test('function f(){return [0, 1, 2, 3, 4, 5, 6, 7, 8].slice(-3, -1)}', [6, 7]) - self._test('function f(){return "012345678".slice()}', '012345678') - self._test('function f(){return "012345678".slice(0)}', '012345678') - self._test('function f(){return "012345678".slice(5)}', '5678') - self._test('function f(){return "012345678".slice(99)}', '') - self._test('function f(){return "012345678".slice(-2)}', '78') - self._test('function f(){return "012345678".slice(-99)}', '012345678') - self._test('function f(){return "012345678".slice(0, 0)}', '') - self._test('function f(){return "012345678".slice(1, 0)}', '') - self._test('function f(){return "012345678".slice(0, 1)}', '0') - self._test('function f(){return "012345678".slice(3, 6)}', '345') - self._test('function f(){return "012345678".slice(1, -1)}', '1234567') - self._test('function f(){return "012345678".slice(-1, 1)}', '') - self._test('function f(){return "012345678".slice(-3, -1)}', '67') - - def test_splice(self): - self._test('function f(){var T = ["0", "1", "2"]; T["splice"](2, 1, "0")[0]; return T }', ['0', '1', '0']) - - def test_pop(self): - # pop - self._test('function f(){var a = [0, 1, 2, 3, 4, 5, 6, 7, 8]; return [a.pop(), a]}', - [8, [0, 1, 2, 3, 4, 5, 6, 7]]) - self._test('function f(){return [].pop()}', JS_Undefined) - # push - self._test('function f(){var a = [0, 1, 2]; return [a.push(3, 4), a]}', - [5, [0, 1, 2, 3, 4]]) - self._test('function f(){var a = [0, 1, 2]; return [a.push(), a]}', - [3, [0, 1, 2]]) - - def test_shift(self): - # shift - self._test('function f(){var a = [0, 1, 2, 3, 4, 5, 6, 7, 8]; return [a.shift(), a]}', - [0, [1, 2, 3, 4, 5, 6, 7, 8]]) - self._test('function f(){return [].shift()}', JS_Undefined) - # unshift - self._test('function f(){var a = [0, 1, 2]; return [a.unshift(3, 4), a]}', - [5, [3, 4, 0, 1, 2]]) - self._test('function f(){var a = [0, 1, 2]; return [a.unshift(), a]}', - [3, [0, 1, 2]]) - - def test_forEach(self): - self._test('function f(){var ret = []; var l = [4, 2]; ' - 'var log = function(e,i,a){ret.push([e,i,a]);}; ' - 'l.forEach(log); ' - 'return [ret.length, ret[0][0], ret[1][1], ret[0][2]]}', - [2, 4, 1, [4, 2]]) - self._test('function f(){var ret = []; var l = [4, 2]; ' - 'var log = function(e,i,a){this.push([e,i,a]);}; ' - 'l.forEach(log, ret); ' - 'return [ret.length, ret[0][0], ret[1][1], ret[0][2]]}', - [2, 4, 1, [4, 2]]) - - def test_extract_function(self): - jsi = JSInterpreter('function a(b) { return b + 1; }') - func = jsi.extract_function('a') - self.assertEqual(func([2]), 3) - - def test_extract_function_with_global_stack(self): - jsi = JSInterpreter('function c(d) { return d + e + f + g; }') - func = jsi.extract_function('c', {'e': 10}, {'f': 100, 'g': 1000}) - self.assertEqual(func([1]), 1111) if __name__ == '__main__': diff --git a/test/test_youtube_signature.py b/test/test_youtube_signature.py index 98221b9c2..cc18d0f7b 100644 --- a/test/test_youtube_signature.py +++ b/test/test_youtube_signature.py @@ -1,5 +1,4 @@ #!/usr/bin/env python -# coding: utf-8 from __future__ import unicode_literals @@ -13,7 +12,6 @@ import re import string from youtube_dl.compat import ( - compat_contextlib_suppress, compat_open as open, compat_str, compat_urlretrieve, @@ -52,93 +50,23 @@ _SIG_TESTS = [ ( 'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflBb0OQx.js', 84, - '123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQ0STUVWXYZ!"#$%&\'()*+,@./:;<=>', + '123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQ0STUVWXYZ!"#$%&\'()*+,@./:;<=>' ), ( 'https://s.ytimg.com/yts/jsbin/html5player-en_US-vfl9FYC6l.js', 83, - '123456789abcdefghijklmnopqr0tuvwxyzABCDETGHIJKLMNOPQRS>UVWXYZ!"#$%&\'()*+,-./:;<=F', + '123456789abcdefghijklmnopqr0tuvwxyzABCDETGHIJKLMNOPQRS>UVWXYZ!"#$%&\'()*+,-./:;<=F' ), ( 'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflCGk6yw/html5player.js', '4646B5181C6C3020DF1D9C7FCFEA.AD80ABF70C39BD369CCCAE780AFBB98FA6B6CB42766249D9488C288', - '82C8849D94266724DC6B6AF89BBFA087EACCD963.B93C07FBA084ACAEFCF7C9D1FD0203C6C1815B6B', + '82C8849D94266724DC6B6AF89BBFA087EACCD963.B93C07FBA084ACAEFCF7C9D1FD0203C6C1815B6B' ), ( 'https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js', '312AA52209E3623129A412D56A40F11CB0AF14AE.3EE09501CB14E3BCDC3B2AE808BF3F1D14E7FBF12', '112AA5220913623229A412D56A40F11CB0AF14AE.3EE0950FCB14EEBCDC3B2AE808BF331D14E7FBF3', - ), - ( - 'https://www.youtube.com/s/player/6ed0d907/player_ias.vflset/en_US/base.js', - '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', - 'AOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL2QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0', - ), - ( - 'https://www.youtube.com/s/player/3bb1f723/player_ias.vflset/en_US/base.js', - '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', - 'MyOSJXtKI3m-uME_jv7-pT12gOFC02RFkGoqWpzE0Cs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', - ), - ( - 'https://www.youtube.com/s/player/2f1832d2/player_ias.vflset/en_US/base.js', - '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', - '0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xxAj7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJ2OySqa0q', - ), - ( - 'https://www.youtube.com/s/player/643afba4/tv-player-ias.vflset/tv-player-ias.js', - '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', - 'AAOAOq0QJ8wRAIgXmPlOPSBkkUs1bYFYlJCfe29xx8j7vgpDL0QwbdV06sCIEzpWqMGkFR20CFOS21Tp-7vj_EMu-m37KtXJoOy1', - ), - ( - 'https://www.youtube.com/s/player/363db69b/player_ias.vflset/en_US/base.js', - '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', - '0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpz2ICs6EVdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', - ), - ( - 'https://www.youtube.com/s/player/363db69b/player_ias_tce.vflset/en_US/base.js', - '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', - '0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpz2ICs6EVdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', - ), - ( - 'https://www.youtube.com/s/player/4fcd6e4a/player_ias.vflset/en_US/base.js', - '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', - 'wAOAOq0QJ8ARAIgXmPlOPSBkkUs1bYFYlJCfe29xx8q7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0', - ), - ( - 'https://www.youtube.com/s/player/4fcd6e4a/player_ias_tce.vflset/en_US/base.js', - '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', - 'wAOAOq0QJ8ARAIgXmPlOPSBkkUs1bYFYlJCfe29xx8q7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0', - ), - ( - 'https://www.youtube.com/s/player/20830619/player_ias.vflset/en_US/base.js', - '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', - '7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw', - ), - ( - 'https://www.youtube.com/s/player/20830619/player_ias_tce.vflset/en_US/base.js', - '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', - '7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw', - ), - ( - 'https://www.youtube.com/s/player/20830619/player-plasma-ias-phone-en_US.vflset/base.js', - '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', - '7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw', - ), - ( - 'https://www.youtube.com/s/player/20830619/player-plasma-ias-tablet-en_US.vflset/base.js', - '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', - '7AOq0QJ8wRAIgXmPlOPSBkkAs1bYFYlJCfe29xx8jOv1pDL0Q2bdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_EMu-m37KtXJoOySqa0qaw', - ), - ( - 'https://www.youtube.com/s/player/8a8ac953/player_ias_tce.vflset/en_US/base.js', - '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', - 'IAOAOq0QJ8wRAAgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_E2u-m37KtXJoOySqa0', - ), - ( - 'https://www.youtube.com/s/player/8a8ac953/tv-player-es6.vflset/tv-player-es6.js', - '2aq0aqSyOoJXtK73m-uME_jv7-pT15gOFC02RFkGMqWpzEICs69VdbwQ0LDp1v7j8xx92efCJlYFYb1sUkkBSPOlPmXgIARw8JQ0qOAOAA', - 'IAOAOq0QJ8wRAAgXmPlOPSBkkUs1bYFYlJCfe29xx8j7v1pDL0QwbdV96sCIEzpWqMGkFR20CFOg51Tp-7vj_E2u-m37KtXJoOySqa0', - ), + ) ] _NSIG_TESTS = [ @@ -208,16 +136,12 @@ _NSIG_TESTS = [ ), ( 'https://www.youtube.com/s/player/c57c113c/player_ias.vflset/en_US/base.js', - 'M92UUMHa8PdvPd3wyM', '3hPqLJsiNZx7yA', + '-Txvy6bT5R6LqgnQNx', 'dcklJCnRUHbgSg', ), ( 'https://www.youtube.com/s/player/5a3b6271/player_ias.vflset/en_US/base.js', 'B2j7f_UPT4rfje85Lu_e', 'm5DmNymaGQ5RdQ', ), - ( - 'https://www.youtube.com/s/player/7a062b77/player_ias.vflset/en_US/base.js', - 'NRcE3y3mVtm_cV-W', 'VbsCYUATvqlt5w', - ), ( 'https://www.youtube.com/s/player/dac945fd/player_ias.vflset/en_US/base.js', 'o8BkRxXhuYsBCWi6RplPdP', '3Lx32v_hmzTm6A', @@ -228,11 +152,7 @@ _NSIG_TESTS = [ ), ( 'https://www.youtube.com/s/player/cfa9e7cb/player_ias.vflset/en_US/base.js', - 'aCi3iElgd2kq0bxVbQ', 'QX1y8jGb2IbZ0w', - ), - ( - 'https://www.youtube.com/s/player/8c7583ff/player_ias.vflset/en_US/base.js', - '1wWCVpRR96eAmMI87L', 'KSkWAVv1ZQxC3A', + 'qO0NiMtYQ7TeJnfFG2', 'k9cuJDHNS5O7kQ', ), ( 'https://www.youtube.com/s/player/b7910ca8/player_ias.vflset/en_US/base.js', @@ -246,110 +166,6 @@ _NSIG_TESTS = [ 'https://www.youtube.com/s/player/b22ef6e7/player_ias.vflset/en_US/base.js', 'b6HcntHGkvBLk_FRf', 'kNPW6A7FyP2l8A', ), - ( - 'https://www.youtube.com/s/player/3400486c/player_ias.vflset/en_US/base.js', - 'lL46g3XifCKUZn1Xfw', 'z767lhet6V2Skl', - ), - ( - 'https://www.youtube.com/s/player/5604538d/player_ias.vflset/en_US/base.js', - '7X-he4jjvMx7BCX', 'sViSydX8IHtdWA', - ), - ( - 'https://www.youtube.com/s/player/20dfca59/player_ias.vflset/en_US/base.js', - '-fLCxedkAk4LUTK2', 'O8kfRq1y1eyHGw', - ), - ( - 'https://www.youtube.com/s/player/b12cc44b/player_ias.vflset/en_US/base.js', - 'keLa5R2U00sR9SQK', 'N1OGyujjEwMnLw', - ), - ( - 'https://www.youtube.com/s/player/3bb1f723/player_ias.vflset/en_US/base.js', - 'gK15nzVyaXE9RsMP3z', 'ZFFWFLPWx9DEgQ', - ), - ( - 'https://www.youtube.com/s/player/f8f53e1a/player_ias.vflset/en_US/base.js', - 'VTQOUOv0mCIeJ7i8kZB', 'kcfD8wy0sNLyNQ', - ), - ( - 'https://www.youtube.com/s/player/2f1832d2/player_ias.vflset/en_US/base.js', - 'YWt1qdbe8SAfkoPHW5d', 'RrRjWQOJmBiP', - ), - ( - 'https://www.youtube.com/s/player/9c6dfc4a/player_ias.vflset/en_US/base.js', - 'jbu7ylIosQHyJyJV', 'uwI0ESiynAmhNg', - ), - ( - 'https://www.youtube.com/s/player/f6e09c70/player_ias.vflset/en_US/base.js', - 'W9HJZKktxuYoDTqW', 'jHbbkcaxm54', - ), - ( - 'https://www.youtube.com/s/player/f6e09c70/player_ias_tce.vflset/en_US/base.js', - 'W9HJZKktxuYoDTqW', 'jHbbkcaxm54', - ), - ( - 'https://www.youtube.com/s/player/e7567ecf/player_ias_tce.vflset/en_US/base.js', - 'Sy4aDGc0VpYRR9ew_', '5UPOT1VhoZxNLQ', - ), - ( - 'https://www.youtube.com/s/player/d50f54ef/player_ias_tce.vflset/en_US/base.js', - 'Ha7507LzRmH3Utygtj', 'XFTb2HoeOE5MHg', - ), - ( - 'https://www.youtube.com/s/player/074a8365/player_ias_tce.vflset/en_US/base.js', - 'Ha7507LzRmH3Utygtj', 'ufTsrE0IVYrkl8v', - ), - ( - 'https://www.youtube.com/s/player/643afba4/player_ias.vflset/en_US/base.js', - 'N5uAlLqm0eg1GyHO', 'dCBQOejdq5s-ww', - ), - ( - 'https://www.youtube.com/s/player/69f581a5/tv-player-ias.vflset/tv-player-ias.js', - '-qIP447rVlTTwaZjY', 'KNcGOksBAvwqQg', - ), - ( - 'https://www.youtube.com/s/player/643afba4/tv-player-ias.vflset/tv-player-ias.js', - 'ir9-V6cdbCiyKxhr', '2PL7ZDYAALMfmA', - ), - ( - 'https://www.youtube.com/s/player/643afba4/player_ias.vflset/en_US/base.js', - 'ir9-V6cdbCiyKxhr', '2PL7ZDYAALMfmA', - ), - ( - 'https://www.youtube.com/s/player/363db69b/player_ias.vflset/en_US/base.js', - 'eWYu5d5YeY_4LyEDc', 'XJQqf-N7Xra3gg', - ), - ( - 'https://www.youtube.com/s/player/4fcd6e4a/player_ias.vflset/en_US/base.js', - 'o_L251jm8yhZkWtBW', 'lXoxI3XvToqn6A', - ), - ( - 'https://www.youtube.com/s/player/4fcd6e4a/tv-player-ias.vflset/tv-player-ias.js', - 'o_L251jm8yhZkWtBW', 'lXoxI3XvToqn6A', - ), - ( - 'https://www.youtube.com/s/player/20830619/tv-player-ias.vflset/tv-player-ias.js', - 'ir9-V6cdbCiyKxhr', '9YE85kNjZiS4', - ), - ( - 'https://www.youtube.com/s/player/20830619/player-plasma-ias-phone-en_US.vflset/base.js', - 'ir9-V6cdbCiyKxhr', '9YE85kNjZiS4', - ), - ( - 'https://www.youtube.com/s/player/20830619/player-plasma-ias-tablet-en_US.vflset/base.js', - 'ir9-V6cdbCiyKxhr', '9YE85kNjZiS4', - ), - ( - 'https://www.youtube.com/s/player/8a8ac953/player_ias_tce.vflset/en_US/base.js', - 'MiBYeXx_vRREbiCCmh', 'RtZYMVvmkE0JE', - ), - ( - 'https://www.youtube.com/s/player/8a8ac953/tv-player-es6.vflset/tv-player-es6.js', - 'MiBYeXx_vRREbiCCmh', 'RtZYMVvmkE0JE', - ), - ( - 'https://www.youtube.com/s/player/aa3fc80b/player_ias.vflset/en_US/base.js', - '0qY9dal2uzOnOGwa-48hha', 'VSh1KDfQMk-eag', - ), ] @@ -362,8 +178,6 @@ class TestPlayerInfo(unittest.TestCase): ('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-phone-en_US.vflset/base.js', '64dddad9'), ('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-phone-de_DE.vflset/base.js', '64dddad9'), ('https://www.youtube.com/s/player/64dddad9/player-plasma-ias-tablet-en_US.vflset/base.js', '64dddad9'), - ('https://www.youtube.com/s/player/e7567ecf/player_ias_tce.vflset/en_US/base.js', 'e7567ecf'), - ('https://www.youtube.com/s/player/643afba4/tv-player-ias.vflset/tv-player-ias.js', '643afba4'), # obsolete ('https://www.youtube.com/yts/jsbin/player_ias-vfle4-e03/en_US/base.js', 'vfle4-e03'), ('https://www.youtube.com/yts/jsbin/player_ias-vfl49f_g4/en_US/base.js', 'vfl49f_g4'), @@ -373,9 +187,8 @@ class TestPlayerInfo(unittest.TestCase): ('https://s.ytimg.com/yts/jsbin/html5player-en_US-vflXGBaUN.js', 'vflXGBaUN'), ('https://s.ytimg.com/yts/jsbin/html5player-en_US-vflKjOTVq/html5player.js', 'vflKjOTVq'), ) - ie = YoutubeIE(FakeYDL({'cachedir': False})) for player_url, expected_player_id in PLAYER_URLS: - player_id = ie._extract_player_info(player_url) + player_id = YoutubeIE._extract_player_info(player_url) self.assertEqual(player_id, expected_player_id) @@ -387,19 +200,21 @@ class TestSignature(unittest.TestCase): os.mkdir(self.TESTDATA_DIR) def tearDown(self): - with compat_contextlib_suppress(OSError): + try: for f in os.listdir(self.TESTDATA_DIR): os.remove(f) + except OSError: + pass def t_factory(name, sig_func, url_pattern): def make_tfunc(url, sig_input, expected_sig): m = url_pattern.match(url) - assert m, '{0!r} should follow URL format'.format(url) - test_id = re.sub(r'[/.-]', '_', m.group('id') or m.group('compat_id')) + assert m, '%r should follow URL format' % url + test_id = m.group('id') def test_func(self): - basename = 'player-{0}.js'.format(test_id) + basename = 'player-{0}-{1}.js'.format(name, test_id) fn = os.path.join(self.TESTDATA_DIR, basename) if not os.path.exists(fn): @@ -414,7 +229,7 @@ def t_factory(name, sig_func, url_pattern): def signature(jscode, sig_input): - func = YoutubeIE(FakeYDL({'cachedir': False}))._parse_sig_js(jscode) + func = YoutubeIE(FakeYDL())._parse_sig_js(jscode) src_sig = ( compat_str(string.printable[:sig_input]) if isinstance(sig_input, int) else sig_input) @@ -422,23 +237,17 @@ def signature(jscode, sig_input): def n_sig(jscode, sig_input): - ie = YoutubeIE(FakeYDL({'cachedir': False})) - jsi = JSInterpreter(jscode) - jsi, _, func_code = ie._extract_n_function_code_jsi(sig_input, jsi) - return ie._extract_n_function_from_code(jsi, func_code)(sig_input) + funcname = YoutubeIE(FakeYDL())._extract_n_function_name(jscode) + return JSInterpreter(jscode).call_function(funcname, sig_input) make_sig_test = t_factory( - 'signature', signature, - re.compile(r'''(?x) - .+/(?P
html5)?player(?(h5)(?:-en_US)?-|/)(?P[a-zA-Z0-9/._-]+) - (?(h5)/(?:watch_as3|html5player))?\.js$ - ''')) + 'signature', signature, re.compile(r'.*-(?P[a-zA-Z0-9_-]+)(?:/watch_as3|/html5player)?\.[a-z]+$')) for test_spec in _SIG_TESTS: make_sig_test(*test_spec) make_nsig_test = t_factory( - 'nsig', n_sig, re.compile(r'.+/player/(?P[a-zA-Z0-9_/.-]+)\.js$')) + 'nsig', n_sig, re.compile(r'.+/player/(?P[a-zA-Z0-9_-]+)/.+.js$')) for test_spec in _NSIG_TESTS: make_nsig_test(*test_spec) diff --git a/youtube_dl/YoutubeDL.py b/youtube_dl/YoutubeDL.py index 8367b6e53..9e5620eef 100755 --- a/youtube_dl/YoutubeDL.py +++ b/youtube_dl/YoutubeDL.py @@ -540,14 +540,10 @@ class YoutubeDL(object): """Print message to stdout if not in quiet mode.""" return self.to_stdout(message, skip_eol, check_quiet=True) - def _write_string(self, s, out=None, only_once=False, _cache=set()): - if only_once and s in _cache: - return + def _write_string(self, s, out=None): write_string(s, out=out, encoding=self.params.get('encoding')) - if only_once: - _cache.add(s) - def to_stdout(self, message, skip_eol=False, check_quiet=False, only_once=False): + def to_stdout(self, message, skip_eol=False, check_quiet=False): """Print message to stdout if not in quiet mode.""" if self.params.get('logger'): self.params['logger'].debug(message) @@ -556,9 +552,9 @@ class YoutubeDL(object): terminator = ['\n', ''][skip_eol] output = message + terminator - self._write_string(output, self._screen_file, only_once=only_once) + self._write_string(output, self._screen_file) - def to_stderr(self, message, only_once=False): + def to_stderr(self, message): """Print message to stderr.""" assert isinstance(message, compat_str) if self.params.get('logger'): @@ -566,7 +562,7 @@ class YoutubeDL(object): else: message = self._bidi_workaround(message) output = message + '\n' - self._write_string(output, self._err_file, only_once=only_once) + self._write_string(output, self._err_file) def to_console_title(self, message): if not self.params.get('consoletitle', False): @@ -645,11 +641,18 @@ class YoutubeDL(object): raise DownloadError(message, exc_info) self._download_retcode = 1 - def report_warning(self, message, only_once=False): + def report_warning(self, message, only_once=False, _cache={}): ''' Print the message to stderr, it will be prefixed with 'WARNING:' If stderr is a tty file the 'WARNING:' will be colored ''' + if only_once: + m_hash = hash((self, message)) + m_cnt = _cache.setdefault(m_hash, 0) + _cache[m_hash] = m_cnt + 1 + if m_cnt > 0: + return + if self.params.get('logger') is not None: self.params['logger'].warning(message) else: @@ -660,7 +663,7 @@ class YoutubeDL(object): else: _msg_header = 'WARNING:' warning_message = '%s %s' % (_msg_header, message) - self.to_stderr(warning_message, only_once=only_once) + self.to_stderr(warning_message) def report_error(self, message, *args, **kwargs): ''' @@ -674,16 +677,6 @@ class YoutubeDL(object): kwargs['message'] = '%s %s' % (_msg_header, message) self.trouble(*args, **kwargs) - def write_debug(self, message, only_once=False): - '''Log debug message or Print message to stderr''' - if not self.params.get('verbose', False): - return - message = '[debug] {0}'.format(message) - if self.params.get('logger'): - self.params['logger'].debug(message) - else: - self.to_stderr(message, only_once) - def report_unscoped_cookies(self, *args, **kwargs): # message=None, tb=False, is_error=False if len(args) <= 2: @@ -2521,7 +2514,7 @@ class YoutubeDL(object): self.get_encoding())) write_string(encoding_str, encoding=None) - writeln_debug = lambda *s: self.write_debug(''.join(s)) + writeln_debug = lambda *s: self._write_string('[debug] %s\n' % (''.join(s), )) writeln_debug('youtube-dl version ', __version__) if _LAZY_LOADER: writeln_debug('Lazy loading extractors enabled') diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index 3c1272e7b..06bdfb689 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -18,7 +18,7 @@ from .compat import ( compat_getpass, compat_register_utf8, compat_shlex_split, - _workaround_optparse_bug9161, + workaround_optparse_bug9161, ) from .utils import ( _UnsafeExtensionError, @@ -50,7 +50,7 @@ def _real_main(argv=None): # Compatibility fix for Windows compat_register_utf8() - _workaround_optparse_bug9161() + workaround_optparse_bug9161() setproctitle('youtube-dl') diff --git a/youtube_dl/cache.py b/youtube_dl/cache.py index eb0a729c2..54123da0e 100644 --- a/youtube_dl/cache.py +++ b/youtube_dl/cache.py @@ -1,4 +1,3 @@ -# coding: utf-8 from __future__ import unicode_literals import errno @@ -11,14 +10,12 @@ import traceback from .compat import ( compat_getenv, compat_open as open, - compat_os_makedirs, ) from .utils import ( error_to_compat_str, - escape_rfc3986, expand_path, is_outdated_version, - traverse_obj, + try_get, write_json_file, ) from .version import __version__ @@ -33,35 +30,23 @@ class Cache(object): def __init__(self, ydl): self._ydl = ydl - def _write_debug(self, *args, **kwargs): - self._ydl.write_debug(*args, **kwargs) - - def _report_warning(self, *args, **kwargs): - self._ydl.report_warning(*args, **kwargs) - - def _to_screen(self, *args, **kwargs): - self._ydl.to_screen(*args, **kwargs) - - def _get_param(self, k, default=None): - return self._ydl.params.get(k, default) - def _get_root_dir(self): - res = self._get_param('cachedir') + res = self._ydl.params.get('cachedir') if res is None: cache_root = compat_getenv('XDG_CACHE_HOME', '~/.cache') res = os.path.join(cache_root, self._YTDL_DIR) return expand_path(res) def _get_cache_fn(self, section, key, dtype): - assert re.match(r'^[\w.-]+$', section), \ + assert re.match(r'^[a-zA-Z0-9_.-]+$', section), \ 'invalid section %r' % section - key = escape_rfc3986(key, safe='').replace('%', ',') # encode non-ascii characters + assert re.match(r'^[a-zA-Z0-9_.-]+$', key), 'invalid key %r' % key return os.path.join( self._get_root_dir(), section, '%s.%s' % (key, dtype)) @property def enabled(self): - return self._get_param('cachedir') is not False + return self._ydl.params.get('cachedir') is not False def store(self, section, key, data, dtype='json'): assert dtype in ('json',) @@ -71,75 +56,61 @@ class Cache(object): fn = self._get_cache_fn(section, key, dtype) try: - compat_os_makedirs(os.path.dirname(fn), exist_ok=True) - self._write_debug('Saving {section}.{key} to cache'.format(section=section, key=key)) + try: + os.makedirs(os.path.dirname(fn)) + except OSError as ose: + if ose.errno != errno.EEXIST: + raise write_json_file({self._VERSION_KEY: __version__, 'data': data}, fn) except Exception: tb = traceback.format_exc() - self._report_warning('Writing cache to {fn!r} failed: {tb}'.format(fn=fn, tb=tb)) - - def clear(self, section, key, dtype='json'): - - if not self.enabled: - return - - fn = self._get_cache_fn(section, key, dtype) - self._write_debug('Clearing {section}.{key} from cache'.format(section=section, key=key)) - try: - os.remove(fn) - except Exception as e: - if getattr(e, 'errno') == errno.ENOENT: - # file not found - return - tb = traceback.format_exc() - self._report_warning('Clearing cache from {fn!r} failed: {tb}'.format(fn=fn, tb=tb)) + self._ydl.report_warning( + 'Writing cache to %r failed: %s' % (fn, tb)) def _validate(self, data, min_ver): - version = traverse_obj(data, self._VERSION_KEY) + version = try_get(data, lambda x: x[self._VERSION_KEY]) if not version: # Backward compatibility data, version = {'data': data}, self._DEFAULT_VERSION if not is_outdated_version(version, min_ver or '0', assume_new=False): return data['data'] - self._write_debug('Discarding old cache from version {version} (needs {min_ver})'.format(version=version, min_ver=min_ver)) + self._ydl.to_screen( + 'Discarding old cache from version {version} (needs {min_ver})'.format(**locals())) - def load(self, section, key, dtype='json', default=None, **kw_min_ver): + def load(self, section, key, dtype='json', default=None, min_ver=None): assert dtype in ('json',) - min_ver = kw_min_ver.get('min_ver') if not self.enabled: return default cache_fn = self._get_cache_fn(section, key, dtype) try: - with open(cache_fn, encoding='utf-8') as cachef: - self._write_debug('Loading {section}.{key} from cache'.format(section=section, key=key), only_once=True) - return self._validate(json.load(cachef), min_ver) - except (ValueError, KeyError): try: - file_size = 'size: %d' % os.path.getsize(cache_fn) - except (OSError, IOError) as oe: - file_size = error_to_compat_str(oe) - self._report_warning('Cache retrieval from %s failed (%s)' % (cache_fn, file_size)) - except Exception as e: - if getattr(e, 'errno') == errno.ENOENT: - # no cache available - return - self._report_warning('Cache retrieval from %s failed' % (cache_fn,)) + with open(cache_fn, 'r', encoding='utf-8') as cachef: + return self._validate(json.load(cachef), min_ver) + except ValueError: + try: + file_size = os.path.getsize(cache_fn) + except (OSError, IOError) as oe: + file_size = error_to_compat_str(oe) + self._ydl.report_warning( + 'Cache retrieval from %s failed (%s)' % (cache_fn, file_size)) + except IOError: + pass # No cache available return default def remove(self): if not self.enabled: - self._to_screen('Cache is disabled (Did you combine --no-cache-dir and --rm-cache-dir?)') + self._ydl.to_screen('Cache is disabled (Did you combine --no-cache-dir and --rm-cache-dir?)') return cachedir = self._get_root_dir() if not any((term in cachedir) for term in ('cache', 'tmp')): - raise Exception('Not removing directory %s - this does not look like a cache dir' % (cachedir,)) + raise Exception('Not removing directory %s - this does not look like a cache dir' % cachedir) - self._to_screen( - 'Removing cache dir %s .' % (cachedir,), skip_eol=True, ), + self._ydl.to_screen( + 'Removing cache dir %s .' % cachedir, skip_eol=True) if os.path.exists(cachedir): - self._to_screen('.', skip_eol=True) + self._ydl.to_screen('.', skip_eol=True) shutil.rmtree(cachedir) - self._to_screen('.') + self._ydl.to_screen('.') diff --git a/youtube_dl/casefold.py b/youtube_dl/casefold.py index 712b2e7fa..ad9c66f8e 100644 --- a/youtube_dl/casefold.py +++ b/youtube_dl/casefold.py @@ -10,10 +10,9 @@ from .compat import ( # https://github.com/unicode-org/icu/blob/main/icu4c/source/data/unidata/CaseFolding.txt # In case newly foldable Unicode characters are defined, paste the new version # of the text inside the ''' marks. -# The text is expected to have only blank lines and lines with 1st character #, +# The text is expected to have only blank lines andlines with 1st character #, # all ignored, and fold definitions like this: -# `from_hex_code; status; space_separated_to_hex_code_list; comment` -# Only `status` C/F are used. +# `from_hex_code; space_separated_to_hex_code_list; comment` _map_str = ''' # CaseFolding-15.0.0.txt @@ -1658,6 +1657,11 @@ _map = dict( del _map_str -def _casefold(s): +def casefold(s): assert isinstance(s, compat_str) return ''.join((_map.get(c, c) for c in s)) + + +__all__ = [ + 'casefold', +] diff --git a/youtube_dl/compat.py b/youtube_dl/compat.py index ebe22bdf9..ed1a33cf2 100644 --- a/youtube_dl/compat.py +++ b/youtube_dl/compat.py @@ -16,6 +16,7 @@ import os import platform import re import shlex +import shutil import socket import struct import subprocess @@ -23,15 +24,11 @@ import sys import types import xml.etree.ElementTree -_IDENTITY = lambda x: x - # naming convention # 'compat_' + Python3_name.replace('.', '_') # other aliases exist for convenience and/or legacy -# wrap disposable test values in type() to reclaim storage -# deal with critical unicode/str things first: -# compat_str, compat_basestring, compat_chr +# deal with critical unicode/str things first try: # Python 2 compat_str, compat_basestring, compat_chr = ( @@ -42,23 +39,18 @@ except NameError: str, (str, bytes), chr ) - -# compat_casefold +# casefold try: compat_str.casefold compat_casefold = lambda s: s.casefold() except AttributeError: - from .casefold import _casefold as compat_casefold + from .casefold import casefold as compat_casefold - -# compat_collections_abc try: import collections.abc as compat_collections_abc except ImportError: import collections as compat_collections_abc - -# compat_urllib_request try: import urllib.request as compat_urllib_request except ImportError: # Python 2 @@ -87,15 +79,11 @@ except TypeError: _add_init_method_arg(compat_urllib_request.Request) del _add_init_method_arg - -# compat_urllib_error try: import urllib.error as compat_urllib_error except ImportError: # Python 2 import urllib2 as compat_urllib_error - -# compat_urllib_parse try: import urllib.parse as compat_urllib_parse except ImportError: # Python 2 @@ -110,23 +98,17 @@ except ImportError: # Python 2 compat_urlparse = compat_urllib_parse compat_urllib_parse_urlparse = compat_urllib_parse.urlparse - -# compat_urllib_response try: import urllib.response as compat_urllib_response except ImportError: # Python 2 import urllib as compat_urllib_response - -# compat_urllib_response.addinfourl try: compat_urllib_response.addinfourl.status except AttributeError: # .getcode() is deprecated in Py 3. compat_urllib_response.addinfourl.status = property(lambda self: self.getcode()) - -# compat_http_cookiejar try: import http.cookiejar as compat_cookiejar except ImportError: # Python 2 @@ -145,16 +127,12 @@ else: compat_cookiejar_Cookie = compat_cookiejar.Cookie compat_http_cookiejar_Cookie = compat_cookiejar_Cookie - -# compat_http_cookies try: import http.cookies as compat_cookies except ImportError: # Python 2 import Cookie as compat_cookies compat_http_cookies = compat_cookies - -# compat_http_cookies_SimpleCookie if sys.version_info[0] == 2 or sys.version_info < (3, 3): class compat_cookies_SimpleCookie(compat_cookies.SimpleCookie): def load(self, rawdata): @@ -177,15 +155,11 @@ else: compat_cookies_SimpleCookie = compat_cookies.SimpleCookie compat_http_cookies_SimpleCookie = compat_cookies_SimpleCookie - -# compat_html_entities, probably useless now try: import html.entities as compat_html_entities except ImportError: # Python 2 import htmlentitydefs as compat_html_entities - -# compat_html_entities_html5 try: # Python >= 3.3 compat_html_entities_html5 = compat_html_entities.html5 except AttributeError: @@ -2434,24 +2408,18 @@ except AttributeError: # Py < 3.1 compat_http_client.HTTPResponse.getcode = lambda self: self.status - -# compat_urllib_HTTPError try: from urllib.error import HTTPError as compat_HTTPError except ImportError: # Python 2 from urllib2 import HTTPError as compat_HTTPError compat_urllib_HTTPError = compat_HTTPError - -# compat_urllib_request_urlretrieve try: from urllib.request import urlretrieve as compat_urlretrieve except ImportError: # Python 2 from urllib import urlretrieve as compat_urlretrieve compat_urllib_request_urlretrieve = compat_urlretrieve - -# compat_html_parser_HTMLParser, compat_html_parser_HTMLParseError try: from HTMLParser import ( HTMLParser as compat_HTMLParser, @@ -2464,33 +2432,22 @@ except ImportError: # Python 3 # HTMLParseError was deprecated in Python 3.3 and removed in # Python 3.5. Introducing dummy exception for Python >3.5 for compatible # and uniform cross-version exception handling - class compat_HTMLParseError(Exception): pass - compat_html_parser_HTMLParser = compat_HTMLParser compat_html_parser_HTMLParseError = compat_HTMLParseError - -# compat_subprocess_get_DEVNULL try: _DEVNULL = subprocess.DEVNULL compat_subprocess_get_DEVNULL = lambda: _DEVNULL except AttributeError: compat_subprocess_get_DEVNULL = lambda: open(os.path.devnull, 'w') - -# compat_http_server try: import http.server as compat_http_server except ImportError: import BaseHTTPServer as compat_http_server - -# compat_urllib_parse_unquote_to_bytes, -# compat_urllib_parse_unquote, compat_urllib_parse_unquote_plus, -# compat_urllib_parse_urlencode, -# compat_urllib_parse_parse_qs try: from urllib.parse import unquote_to_bytes as compat_urllib_parse_unquote_to_bytes from urllib.parse import unquote as compat_urllib_parse_unquote @@ -2498,7 +2455,8 @@ try: from urllib.parse import urlencode as compat_urllib_parse_urlencode from urllib.parse import parse_qs as compat_parse_qs except ImportError: # Python 2 - _asciire = getattr(compat_urllib_parse, '_asciire', None) or re.compile(r'([\x00-\x7f]+)') + _asciire = (compat_urllib_parse._asciire if hasattr(compat_urllib_parse, '_asciire') + else re.compile(r'([\x00-\x7f]+)')) # HACK: The following are the correct unquote_to_bytes, unquote and unquote_plus # implementations from cpython 3.4.3's stdlib. Python 2's version @@ -2566,21 +2524,24 @@ except ImportError: # Python 2 # Possible solutions are to either port it from python 3 with all # the friends or manually ensure input query contains only byte strings. # We will stick with latter thus recursively encoding the whole query. - def compat_urllib_parse_urlencode(query, doseq=0, safe='', encoding='utf-8', errors='strict'): - + def compat_urllib_parse_urlencode(query, doseq=0, encoding='utf-8'): def encode_elem(e): if isinstance(e, dict): e = encode_dict(e) elif isinstance(e, (list, tuple,)): - e = type(e)(encode_elem(el) for el in e) + list_e = encode_list(e) + e = tuple(list_e) if isinstance(e, tuple) else list_e elif isinstance(e, compat_str): - e = e.encode(encoding, errors) + e = e.encode(encoding) return e def encode_dict(d): - return tuple((encode_elem(k), encode_elem(v)) for k, v in d.items()) + return dict((encode_elem(k), encode_elem(v)) for k, v in d.items()) - return compat_urllib_parse._urlencode(encode_elem(query), doseq=doseq).decode('ascii') + def encode_list(l): + return [encode_elem(e) for e in l] + + return compat_urllib_parse._urlencode(encode_elem(query), doseq=doseq) # HACK: The following is the correct parse_qs implementation from cpython 3's stdlib. # Python 2's version is apparently totally broken @@ -2635,61 +2596,8 @@ except ImportError: # Python 2 ('parse_qs', compat_parse_qs)): setattr(compat_urllib_parse, name, fix) - try: - all(chr(i) in b'' for i in range(256)) - except TypeError: - # not all chr(i) are str: patch Python2 quote - - _safemaps = getattr(compat_urllib_parse, '_safemaps', {}) - _always_safe = frozenset(compat_urllib_parse.always_safe) - - def _quote(s, safe='/'): - """quote('abc def') -> 'abc%20def'""" - - if not s and s is not None: # fast path - return s - safe = frozenset(safe) - cachekey = (safe, _always_safe) - try: - safe_map = _safemaps[cachekey] - except KeyError: - safe = _always_safe | safe - safe_map = {} - for i in range(256): - c = chr(i) - safe_map[c] = ( - c if (i < 128 and c in safe) - else b'%{0:02X}'.format(i)) - _safemaps[cachekey] = safe_map - - if safe.issuperset(s): - return s - return ''.join(safe_map[c] for c in s) - - # linked code - def _quote_plus(s, safe=''): - return ( - _quote(s, safe + b' ').replace(b' ', b'+') if b' ' in s - else _quote(s, safe)) - - # linked code - def _urlcleanup(): - if compat_urllib_parse._urlopener: - compat_urllib_parse._urlopener.cleanup() - _safemaps.clear() - compat_urllib_parse.ftpcache.clear() - - for name, fix in ( - ('quote', _quote), - ('quote_plus', _quote_plus), - ('urlcleanup', _urlcleanup)): - setattr(compat_urllib_parse, '_' + name, getattr(compat_urllib_parse, name)) - setattr(compat_urllib_parse, name, fix) - compat_urllib_parse_parse_qs = compat_parse_qs - -# compat_urllib_request_DataHandler try: from urllib.request import DataHandler as compat_urllib_request_DataHandler except ImportError: # Python < 3.4 @@ -2724,20 +2632,16 @@ except ImportError: # Python < 3.4 return compat_urllib_response.addinfourl(io.BytesIO(data), headers, url) - -# compat_xml_etree_ElementTree_ParseError try: from xml.etree.ElementTree import ParseError as compat_xml_parse_error except ImportError: # Python 2.6 from xml.parsers.expat import ExpatError as compat_xml_parse_error compat_xml_etree_ElementTree_ParseError = compat_xml_parse_error - -# compat_xml_etree_ElementTree_Element -_etree = xml.etree.ElementTree +etree = xml.etree.ElementTree -class _TreeBuilder(_etree.TreeBuilder): +class _TreeBuilder(etree.TreeBuilder): def doctype(self, name, pubid, system): pass @@ -2746,7 +2650,7 @@ try: # xml.etree.ElementTree.Element is a method in Python <=2.6 and # the following will crash with: # TypeError: isinstance() arg 2 must be a class, type, or tuple of classes and types - isinstance(None, _etree.Element) + isinstance(None, etree.Element) from xml.etree.ElementTree import Element as compat_etree_Element except TypeError: # Python <=2.6 from xml.etree.ElementTree import _ElementInterface as compat_etree_Element @@ -2754,12 +2658,12 @@ compat_xml_etree_ElementTree_Element = compat_etree_Element if sys.version_info[0] >= 3: def compat_etree_fromstring(text): - return _etree.XML(text, parser=_etree.XMLParser(target=_TreeBuilder())) + return etree.XML(text, parser=etree.XMLParser(target=_TreeBuilder())) else: # python 2.x tries to encode unicode strings with ascii (see the # XMLParser._fixtext method) try: - _etree_iter = _etree.Element.iter + _etree_iter = etree.Element.iter except AttributeError: # Python <=2.6 def _etree_iter(root): for el in root.findall('*'): @@ -2771,29 +2675,27 @@ else: # 2.7 source def _XML(text, parser=None): if not parser: - parser = _etree.XMLParser(target=_TreeBuilder()) + parser = etree.XMLParser(target=_TreeBuilder()) parser.feed(text) return parser.close() def _element_factory(*args, **kwargs): - el = _etree.Element(*args, **kwargs) + el = etree.Element(*args, **kwargs) for k, v in el.items(): if isinstance(v, bytes): el.set(k, v.decode('utf-8')) return el def compat_etree_fromstring(text): - doc = _XML(text, parser=_etree.XMLParser(target=_TreeBuilder(element_factory=_element_factory))) + doc = _XML(text, parser=etree.XMLParser(target=_TreeBuilder(element_factory=_element_factory))) for el in _etree_iter(doc): if el.text is not None and isinstance(el.text, bytes): el.text = el.text.decode('utf-8') return doc - -# compat_xml_etree_register_namespace -try: - compat_etree_register_namespace = _etree.register_namespace -except AttributeError: +if hasattr(etree, 'register_namespace'): + compat_etree_register_namespace = etree.register_namespace +else: def compat_etree_register_namespace(prefix, uri): """Register a namespace prefix. The registry is global, and any existing mapping for either the @@ -2802,16 +2704,14 @@ except AttributeError: attributes in this namespace will be serialized with prefix if possible. ValueError is raised if prefix is reserved or is invalid. """ - if re.match(r'ns\d+$', prefix): - raise ValueError('Prefix format reserved for internal use') - for k, v in list(_etree._namespace_map.items()): + if re.match(r"ns\d+$", prefix): + raise ValueError("Prefix format reserved for internal use") + for k, v in list(etree._namespace_map.items()): if k == uri or v == prefix: - del _etree._namespace_map[k] - _etree._namespace_map[uri] = prefix + del etree._namespace_map[k] + etree._namespace_map[uri] = prefix compat_xml_etree_register_namespace = compat_etree_register_namespace - -# compat_xpath, compat_etree_iterfind if sys.version_info < (2, 7): # Here comes the crazy part: In 2.6, if the xpath is a unicode, # .//node does not match if a node is a direct child of . ! @@ -2998,6 +2898,7 @@ if sys.version_info < (2, 7): def __init__(self, root): self.root = root + ## # Generate all matching objects. def compat_etree_iterfind(elem, path, namespaces=None): @@ -3032,15 +2933,13 @@ if sys.version_info < (2, 7): else: + compat_xpath = lambda xpath: xpath compat_etree_iterfind = lambda element, match: element.iterfind(match) - compat_xpath = _IDENTITY -# compat_os_name compat_os_name = os._name if os.name == 'java' else os.name -# compat_shlex_quote if compat_os_name == 'nt': def compat_shlex_quote(s): return s if re.match(r'^[-_\w./]+$', s) else '"%s"' % s.replace('"', '\\"') @@ -3055,7 +2954,6 @@ else: return "'" + s.replace("'", "'\"'\"'") + "'" -# compat_shlex.split try: args = shlex.split('ไธญๆ–‡') assert (isinstance(args, list) @@ -3071,7 +2969,6 @@ except (AssertionError, UnicodeEncodeError): return list(map(lambda s: s.decode('utf-8'), shlex.split(s, comments, posix))) -# compat_ord def compat_ord(c): if isinstance(c, int): return c @@ -3079,7 +2976,6 @@ def compat_ord(c): return ord(c) -# compat_getenv, compat_os_path_expanduser, compat_setenv if sys.version_info >= (3, 0): compat_getenv = os.getenv compat_expanduser = os.path.expanduser @@ -3167,22 +3063,6 @@ else: compat_os_path_expanduser = compat_expanduser -# compat_os_makedirs -try: - os.makedirs('.', exist_ok=True) - compat_os_makedirs = os.makedirs -except TypeError: # < Py3.2 - from errno import EEXIST as _errno_EEXIST - - def compat_os_makedirs(name, mode=0o777, exist_ok=False): - try: - return os.makedirs(name, mode=mode) - except OSError as ose: - if not (exist_ok and ose.errno == _errno_EEXIST): - raise - - -# compat_os_path_realpath if compat_os_name == 'nt' and sys.version_info < (3, 8): # os.path.realpath on Windows does not follow symbolic links # prior to Python 3.8 (see https://bugs.python.org/issue9949) @@ -3196,7 +3076,6 @@ else: compat_os_path_realpath = compat_realpath -# compat_print if sys.version_info < (3, 0): def compat_print(s): from .utils import preferredencoding @@ -3207,7 +3086,6 @@ else: print(s) -# compat_getpass_getpass if sys.version_info < (3, 0) and sys.platform == 'win32': def compat_getpass(prompt, *args, **kwargs): if isinstance(prompt, compat_str): @@ -3220,42 +3098,36 @@ else: compat_getpass_getpass = compat_getpass -# compat_input try: compat_input = raw_input except NameError: # Python 3 compat_input = input -# compat_kwargs # Python < 2.6.5 require kwargs to be bytes try: - (lambda x: x)(**{'x': 0}) + def _testfunc(x): + pass + _testfunc(**{'x': 0}) except TypeError: def compat_kwargs(kwargs): return dict((bytes(k), v) for k, v in kwargs.items()) else: - compat_kwargs = _IDENTITY + compat_kwargs = lambda kwargs: kwargs -# compat_numeric_types try: compat_numeric_types = (int, float, long, complex) except NameError: # Python 3 compat_numeric_types = (int, float, complex) -# compat_integer_types try: compat_integer_types = (int, long) except NameError: # Python 3 compat_integer_types = (int, ) -# compat_int -compat_int = compat_integer_types[-1] - -# compat_socket_create_connection if sys.version_info < (2, 7): def compat_socket_create_connection(address, timeout, source_address=None): host, port = address @@ -3282,7 +3154,6 @@ else: compat_socket_create_connection = socket.create_connection -# compat_contextlib_suppress try: from contextlib import suppress as compat_contextlib_suppress except ImportError: @@ -3325,12 +3196,12 @@ except AttributeError: # repeated .close() is OK, but just in case with compat_contextlib_suppress(EnvironmentError): f.close() - popen.wait() + popen.wait() # Fix https://github.com/ytdl-org/youtube-dl/issues/4223 # See http://bugs.python.org/issue9161 for what is broken -def _workaround_optparse_bug9161(): +def workaround_optparse_bug9161(): op = optparse.OptionParser() og = optparse.OptionGroup(op, 'foo') try: @@ -3349,10 +3220,9 @@ def _workaround_optparse_bug9161(): optparse.OptionGroup.add_option = _compat_add_option -# compat_shutil_get_terminal_size -try: - from shutil import get_terminal_size as compat_get_terminal_size # Python >= 3.3 -except ImportError: +if hasattr(shutil, 'get_terminal_size'): # Python >= 3.3 + compat_get_terminal_size = shutil.get_terminal_size +else: _terminal_size = collections.namedtuple('terminal_size', ['columns', 'lines']) def compat_get_terminal_size(fallback=(80, 24)): @@ -3382,33 +3252,27 @@ except ImportError: columns = _columns if lines is None or lines <= 0: lines = _lines - return _terminal_size(columns, lines) -compat_shutil_get_terminal_size = compat_get_terminal_size - -# compat_itertools_count try: - type(itertools.count(start=0, step=1)) + itertools.count(start=0, step=1) compat_itertools_count = itertools.count -except TypeError: # Python 2.6 lacks step +except TypeError: # Python 2.6 def compat_itertools_count(start=0, step=1): while True: yield start start += step -# compat_tokenize_tokenize if sys.version_info >= (3, 0): from tokenize import tokenize as compat_tokenize_tokenize else: from tokenize import generate_tokens as compat_tokenize_tokenize -# compat_struct_pack, compat_struct_unpack, compat_Struct try: - type(struct.pack('!I', 0)) + struct.pack('!I', 0) except TypeError: # In Python 2.6 and 2.7.x < 2.7.7, struct requires a bytes argument # See https://bugs.python.org/issue19099 @@ -3440,10 +3304,8 @@ else: compat_Struct = struct.Struct -# builtins returning an iterator - -# compat_map, compat_filter -# supposedly the same versioning as for zip below +# compat_map/filter() returning an iterator, supposedly the +# same versioning as for zip below try: from future_builtins import map as compat_map except ImportError: @@ -3460,7 +3322,6 @@ except ImportError: except ImportError: compat_filter = filter -# compat_zip try: from future_builtins import zip as compat_zip except ImportError: # not 2.6+ or is 3.x @@ -3470,7 +3331,6 @@ except ImportError: # not 2.6+ or is 3.x compat_zip = zip -# compat_itertools_zip_longest # method renamed between Py2/3 try: from itertools import zip_longest as compat_itertools_zip_longest @@ -3478,8 +3338,7 @@ except ImportError: from itertools import izip_longest as compat_itertools_zip_longest -# compat_collections_chain_map -# collections.ChainMap: new class +# new class in collections try: from collections import ChainMap as compat_collections_chain_map # Py3.3's ChainMap is deficient @@ -3535,22 +3394,19 @@ except ImportError: def new_child(self, m=None, **kwargs): m = m or {} m.update(kwargs) - # support inheritance ! - return type(self)(m, *self.maps) + return compat_collections_chain_map(m, *self.maps) @property def parents(self): - return type(self)(*(self.maps[1:])) + return compat_collections_chain_map(*(self.maps[1:])) -# compat_re_Pattern, compat_re_Match # Pythons disagree on the type of a pattern (RegexObject, _sre.SRE_Pattern, Pattern, ...?) compat_re_Pattern = type(re.compile('')) # and on the type of a match compat_re_Match = type(re.match('a', 'a')) -# compat_base64_b64decode if sys.version_info < (3, 3): def compat_b64decode(s, *args, **kwargs): if isinstance(s, compat_str): @@ -3562,7 +3418,6 @@ else: compat_base64_b64decode = compat_b64decode -# compat_ctypes_WINFUNCTYPE if platform.python_implementation() == 'PyPy' and sys.pypy_version_info < (5, 4, 0): # PyPy2 prior to version 5.4.0 expects byte strings as Windows function # names, see the original PyPy issue [1] and the youtube-dl one [2]. @@ -3581,7 +3436,6 @@ else: return ctypes.WINFUNCTYPE(*args, **kwargs) -# compat_open if sys.version_info < (3, 0): # open(file, mode='r', buffering=- 1, encoding=None, errors=None, newline=None, closefd=True) not: opener=None def compat_open(file_, *args, **kwargs): @@ -3609,28 +3463,18 @@ except AttributeError: def compat_datetime_timedelta_total_seconds(td): return (td.microseconds + (td.seconds + td.days * 24 * 3600) * 10**6) / 10**6 - # optional decompression packages -# compat_brotli # PyPi brotli package implements 'br' Content-Encoding try: import brotli as compat_brotli except ImportError: compat_brotli = None -# compat_ncompress # PyPi ncompress package implements 'compress' Content-Encoding try: import ncompress as compat_ncompress except ImportError: compat_ncompress = None -# compat_zstandard -# PyPi zstandard package implements 'zstd' Content-Encoding (RFC 8878 7.2) -try: - import zstandard as compat_zstandard -except ImportError: - compat_zstandard = None - legacy = [ 'compat_HTMLParseError', @@ -3647,7 +3491,6 @@ legacy = [ 'compat_getpass', 'compat_parse_qs', 'compat_realpath', - 'compat_shlex_split', 'compat_urllib_parse_parse_qs', 'compat_urllib_parse_unquote', 'compat_urllib_parse_unquote_plus', @@ -3661,6 +3504,8 @@ legacy = [ __all__ = [ + 'compat_html_parser_HTMLParseError', + 'compat_html_parser_HTMLParser', 'compat_Struct', 'compat_base64_b64decode', 'compat_basestring', @@ -3669,9 +3514,13 @@ __all__ = [ 'compat_chr', 'compat_collections_abc', 'compat_collections_chain_map', + 'compat_datetime_timedelta_total_seconds', + 'compat_http_cookiejar', + 'compat_http_cookiejar_Cookie', + 'compat_http_cookies', + 'compat_http_cookies_SimpleCookie', 'compat_contextlib_suppress', 'compat_ctypes_WINFUNCTYPE', - 'compat_datetime_timedelta_total_seconds', 'compat_etree_fromstring', 'compat_etree_iterfind', 'compat_filter', @@ -3680,16 +3529,9 @@ __all__ = [ 'compat_getpass_getpass', 'compat_html_entities', 'compat_html_entities_html5', - 'compat_html_parser_HTMLParseError', - 'compat_html_parser_HTMLParser', - 'compat_http_cookiejar', - 'compat_http_cookiejar_Cookie', - 'compat_http_cookies', - 'compat_http_cookies_SimpleCookie', 'compat_http_client', 'compat_http_server', 'compat_input', - 'compat_int', 'compat_integer_types', 'compat_itertools_count', 'compat_itertools_zip_longest', @@ -3699,7 +3541,6 @@ __all__ = [ 'compat_numeric_types', 'compat_open', 'compat_ord', - 'compat_os_makedirs', 'compat_os_name', 'compat_os_path_expanduser', 'compat_os_path_realpath', @@ -3709,7 +3550,7 @@ __all__ = [ 'compat_register_utf8', 'compat_setenv', 'compat_shlex_quote', - 'compat_shutil_get_terminal_size', + 'compat_shlex_split', 'compat_socket_create_connection', 'compat_str', 'compat_struct_pack', @@ -3729,5 +3570,5 @@ __all__ = [ 'compat_xml_etree_register_namespace', 'compat_xpath', 'compat_zip', - 'compat_zstandard', + 'workaround_optparse_bug9161', ] diff --git a/youtube_dl/extractor/bokecc.py b/youtube_dl/extractor/bokecc.py index 4b8bef391..6017e8344 100644 --- a/youtube_dl/extractor/bokecc.py +++ b/youtube_dl/extractor/bokecc.py @@ -32,7 +32,7 @@ class BokeCCBaseIE(InfoExtractor): class BokeCCIE(BokeCCBaseIE): - IE_DESC = 'CC่ง†้ข‘' + _IE_DESC = 'CC่ง†้ข‘' _VALID_URL = r'https?://union\.bokecc\.com/playvideo\.bo\?(?P.*)' _TESTS = [{ diff --git a/youtube_dl/extractor/cloudy.py b/youtube_dl/extractor/cloudy.py index d39a9a5c2..85ca20ecc 100644 --- a/youtube_dl/extractor/cloudy.py +++ b/youtube_dl/extractor/cloudy.py @@ -9,7 +9,7 @@ from ..utils import ( class CloudyIE(InfoExtractor): - IE_DESC = 'cloudy.ec' + _IE_DESC = 'cloudy.ec' _VALID_URL = r'https?://(?:www\.)?cloudy\.ec/(?:v/|embed\.php\?.*?\bid=)(?P[A-Za-z0-9]+)' _TESTS = [{ 'url': 'https://www.cloudy.ec/v/af511e2527aac', diff --git a/youtube_dl/extractor/common.py b/youtube_dl/extractor/common.py index a64fcfccc..9b0016d07 100644 --- a/youtube_dl/extractor/common.py +++ b/youtube_dl/extractor/common.py @@ -422,8 +422,6 @@ class InfoExtractor(object): _GEO_COUNTRIES = None _GEO_IP_BLOCKS = None _WORKING = True - # supply this in public subclasses: used in supported sites list, etc - # IE_DESC = 'short description of IE' def __init__(self, downloader=None): """Constructor. Receives an optional downloader.""" @@ -505,7 +503,7 @@ class InfoExtractor(object): if not self._x_forwarded_for_ip: # Geo bypass mechanism is explicitly disabled by user - if not self.get_param('geo_bypass', True): + if not self._downloader.params.get('geo_bypass', True): return if not geo_bypass_context: @@ -527,7 +525,7 @@ class InfoExtractor(object): # Explicit IP block specified by user, use it right away # regardless of whether extractor is geo bypassable or not - ip_block = self.get_param('geo_bypass_ip_block', None) + ip_block = self._downloader.params.get('geo_bypass_ip_block', None) # Otherwise use random IP block from geo bypass context but only # if extractor is known as geo bypassable @@ -538,8 +536,8 @@ class InfoExtractor(object): if ip_block: self._x_forwarded_for_ip = GeoUtils.random_ipv4(ip_block) - if self.get_param('verbose', False): - self.to_screen( + if self._downloader.params.get('verbose', False): + self._downloader.to_screen( '[debug] Using fake IP %s as X-Forwarded-For.' % self._x_forwarded_for_ip) return @@ -548,7 +546,7 @@ class InfoExtractor(object): # Explicit country code specified by user, use it right away # regardless of whether extractor is geo bypassable or not - country = self.get_param('geo_bypass_country', None) + country = self._downloader.params.get('geo_bypass_country', None) # Otherwise use random country code from geo bypass context but # only if extractor is known as geo bypassable @@ -559,8 +557,8 @@ class InfoExtractor(object): if country: self._x_forwarded_for_ip = GeoUtils.random_ipv4(country) - if self.get_param('verbose', False): - self.to_screen( + if self._downloader.params.get('verbose', False): + self._downloader.to_screen( '[debug] Using fake IP %s (%s) as X-Forwarded-For.' % (self._x_forwarded_for_ip, country.upper())) @@ -586,9 +584,9 @@ class InfoExtractor(object): raise ExtractorError('An extractor error has occurred.', cause=e) def __maybe_fake_ip_and_retry(self, countries): - if (not self.get_param('geo_bypass_country', None) + if (not self._downloader.params.get('geo_bypass_country', None) and self._GEO_BYPASS - and self.get_param('geo_bypass', True) + and self._downloader.params.get('geo_bypass', True) and not self._x_forwarded_for_ip and countries): country_code = random.choice(countries) @@ -698,7 +696,7 @@ class InfoExtractor(object): if fatal: raise ExtractorError(errmsg, sys.exc_info()[2], cause=err) else: - self.report_warning(errmsg) + self._downloader.report_warning(errmsg) return False def _download_webpage_handle(self, url_or_request, video_id, note=None, errnote=None, fatal=True, encoding=None, data=None, headers={}, query={}, expected_status=None): @@ -770,11 +768,11 @@ class InfoExtractor(object): webpage_bytes = prefix + webpage_bytes if not encoding: encoding = self._guess_encoding_from_content(content_type, webpage_bytes) - if self.get_param('dump_intermediate_pages', False): + if self._downloader.params.get('dump_intermediate_pages', False): self.to_screen('Dumping request to ' + urlh.geturl()) dump = base64.b64encode(webpage_bytes).decode('ascii') - self.to_screen(dump) - if self.get_param('write_pages', False): + self._downloader.to_screen(dump) + if self._downloader.params.get('write_pages', False): basen = '%s_%s' % (video_id, urlh.geturl()) if len(basen) > 240: h = '___' + hashlib.md5(basen.encode('utf-8')).hexdigest() @@ -976,9 +974,19 @@ class InfoExtractor(object): """Print msg to screen, prefixing it with '[ie_name]'""" self._downloader.to_screen(self.__ie_msg(msg)) - def write_debug(self, msg, only_once=False): + def write_debug(self, msg, only_once=False, _cache=[]): '''Log debug message or Print message to stderr''' - self._downloader.write_debug(self.__ie_msg(msg), only_once=only_once) + if not self.get_param('verbose', False): + return + message = '[debug] ' + self.__ie_msg(msg) + logger = self.get_param('logger') + if logger: + logger.debug(message) + else: + if only_once and hash(message) in _cache: + return + self._downloader.to_stderr(message) + _cache.append(hash(message)) # name, default=None, *args, **kwargs def get_param(self, name, *args, **kwargs): @@ -1074,7 +1082,7 @@ class InfoExtractor(object): if mobj: break - if not self.get_param('no_color') and compat_os_name != 'nt' and sys.stderr.isatty(): + if not self._downloader.params.get('no_color') and compat_os_name != 'nt' and sys.stderr.isatty(): _name = '\033[0;34m%s\033[0m' % name else: _name = name @@ -1092,7 +1100,7 @@ class InfoExtractor(object): elif fatal: raise RegexNotFoundError('Unable to extract %s' % _name) else: - self.report_warning('unable to extract %s' % _name + bug_reports_message()) + self._downloader.report_warning('unable to extract %s' % _name + bug_reports_message()) return None def _search_json(self, start_pattern, string, name, video_id, **kwargs): @@ -1162,7 +1170,7 @@ class InfoExtractor(object): username = None password = None - if self.get_param('usenetrc', False): + if self._downloader.params.get('usenetrc', False): try: netrc_machine = netrc_machine or self._NETRC_MACHINE info = netrc.netrc().authenticators(netrc_machine) @@ -1173,7 +1181,7 @@ class InfoExtractor(object): raise netrc.NetrcParseError( 'No authenticators for %s' % netrc_machine) except (AttributeError, IOError, netrc.NetrcParseError) as err: - self.report_warning( + self._downloader.report_warning( 'parsing .netrc: %s' % error_to_compat_str(err)) return username, password @@ -1210,10 +1218,10 @@ class InfoExtractor(object): """ if self._downloader is None: return None + downloader_params = self._downloader.params - twofactor = self.get_param('twofactor') - if twofactor is not None: - return twofactor + if downloader_params.get('twofactor') is not None: + return downloader_params['twofactor'] return compat_getpass('Type %s and press [Return]: ' % note) @@ -1348,7 +1356,7 @@ class InfoExtractor(object): elif fatal: raise RegexNotFoundError('Unable to extract JSON-LD') else: - self.report_warning('unable to extract JSON-LD %s' % bug_reports_message()) + self._downloader.report_warning('unable to extract JSON-LD %s' % bug_reports_message()) return {} def _json_ld(self, json_ld, video_id, fatal=True, expected_type=None): @@ -1579,7 +1587,7 @@ class InfoExtractor(object): if f.get('vcodec') == 'none': # audio only preference -= 50 - if self.get_param('prefer_free_formats'): + if self._downloader.params.get('prefer_free_formats'): ORDER = ['aac', 'mp3', 'm4a', 'webm', 'ogg', 'opus'] else: ORDER = ['webm', 'opus', 'ogg', 'mp3', 'aac', 'm4a'] @@ -1591,7 +1599,7 @@ class InfoExtractor(object): else: if f.get('acodec') == 'none': # video only preference -= 40 - if self.get_param('prefer_free_formats'): + if self._downloader.params.get('prefer_free_formats'): ORDER = ['flv', 'mp4', 'webm'] else: ORDER = ['webm', 'flv', 'mp4'] @@ -1657,7 +1665,7 @@ class InfoExtractor(object): """ Either "http:" or "https:", depending on the user's preferences """ return ( 'http:' - if self.get_param('prefer_insecure', False) + if self._downloader.params.get('prefer_insecure', False) else 'https:') def _proto_relative_url(self, url, scheme=None): @@ -3162,7 +3170,7 @@ class InfoExtractor(object): # See com/longtailvideo/jwplayer/media/RTMPMediaProvider.as # of jwplayer.flash.swf rtmp_url_parts = re.split( - r'((?:mp4|mp3|flv):)', source_url, maxsplit=1) + r'((?:mp4|mp3|flv):)', source_url, 1) if len(rtmp_url_parts) == 3: rtmp_url, prefix, play_path = rtmp_url_parts a_format.update({ @@ -3189,7 +3197,7 @@ class InfoExtractor(object): if fatal: raise ExtractorError(msg) else: - self.report_warning(msg) + self._downloader.report_warning(msg) return res def _float(self, v, name, fatal=False, **kwargs): @@ -3199,7 +3207,7 @@ class InfoExtractor(object): if fatal: raise ExtractorError(msg) else: - self.report_warning(msg) + self._downloader.report_warning(msg) return res def _set_cookie(self, domain, name, value, expire_time=None, port=None, @@ -3208,12 +3216,12 @@ class InfoExtractor(object): 0, name, value, port, port is not None, domain, True, domain.startswith('.'), path, True, secure, expire_time, discard, None, None, rest) - self.cookiejar.set_cookie(cookie) + self._downloader.cookiejar.set_cookie(cookie) def _get_cookies(self, url): """ Return a compat_cookies_SimpleCookie with the cookies for the url """ req = sanitized_Request(url) - self.cookiejar.add_cookie_header(req) + self._downloader.cookiejar.add_cookie_header(req) return compat_cookies_SimpleCookie(req.get_header('Cookie')) def _apply_first_set_cookie_header(self, url_handle, cookie): @@ -3273,8 +3281,8 @@ class InfoExtractor(object): return not any_restricted def extract_subtitles(self, *args, **kwargs): - if (self.get_param('writesubtitles', False) - or self.get_param('listsubtitles')): + if (self._downloader.params.get('writesubtitles', False) + or self._downloader.params.get('listsubtitles')): return self._get_subtitles(*args, **kwargs) return {} @@ -3295,11 +3303,7 @@ class InfoExtractor(object): """ Merge subtitle dictionaries, language by language. """ # ..., * , target=None - target = kwargs.get('target') - if target is None: - target = dict(subtitle_dict1) - else: - subtitle_dicts = (subtitle_dict1,) + subtitle_dicts + target = kwargs.get('target') or dict(subtitle_dict1) for subtitle_dict in subtitle_dicts: for lang in subtitle_dict: @@ -3307,8 +3311,8 @@ class InfoExtractor(object): return target def extract_automatic_captions(self, *args, **kwargs): - if (self.get_param('writeautomaticsub', False) - or self.get_param('listsubtitles')): + if (self._downloader.params.get('writeautomaticsub', False) + or self._downloader.params.get('listsubtitles')): return self._get_automatic_captions(*args, **kwargs) return {} @@ -3316,9 +3320,9 @@ class InfoExtractor(object): raise NotImplementedError('This method must be implemented by subclasses') def mark_watched(self, *args, **kwargs): - if (self.get_param('mark_watched', False) + if (self._downloader.params.get('mark_watched', False) and (self._get_login_info()[0] is not None - or self.get_param('cookiefile') is not None)): + or self._downloader.params.get('cookiefile') is not None)): self._mark_watched(*args, **kwargs) def _mark_watched(self, *args, **kwargs): @@ -3326,7 +3330,7 @@ class InfoExtractor(object): def geo_verification_headers(self): headers = {} - geo_verification_proxy = self.get_param('geo_verification_proxy') + geo_verification_proxy = self._downloader.params.get('geo_verification_proxy') if geo_verification_proxy: headers['Ytdl-request-proxy'] = geo_verification_proxy return headers diff --git a/youtube_dl/extractor/itv.py b/youtube_dl/extractor/itv.py index 2510ad887..c64af3be6 100644 --- a/youtube_dl/extractor/itv.py +++ b/youtube_dl/extractor/itv.py @@ -35,6 +35,15 @@ from ..utils import ( class ITVBaseIE(InfoExtractor): + def _search_nextjs_data(self, webpage, video_id, **kw): + transform_source = kw.pop('transform_source', None) + fatal = kw.pop('fatal', True) + return self._parse_json( + self._search_regex( + r''']+\bid=('|")__NEXT_DATA__\1[^>]*>(?P[^<]+)''', + webpage, 'next.js data', group='js', fatal=fatal, **kw), + video_id, transform_source=transform_source, fatal=fatal) + def __handle_request_webpage_error(self, err, video_id=None, errnote=None, fatal=True): if errnote is False: return False @@ -100,9 +109,7 @@ class ITVBaseIE(InfoExtractor): class ITVIE(ITVBaseIE): _VALID_URL = r'https?://(?:www\.)?itv\.com/(?:(?Pwatch)|hub)/[^/]+/(?(w)[\w-]+/)(?P\w+)' - IE_DESC = 'ITVX' - _WORKING = False - + _IE_DESC = 'ITVX' _TESTS = [{ 'note': 'Hub URLs redirect to ITVX', 'url': 'https://www.itv.com/hub/liar/2a4547a0012', @@ -263,7 +270,7 @@ class ITVIE(ITVBaseIE): 'ext': determine_ext(href, 'vtt'), }) - next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default={}) + next_data = self._search_nextjs_data(webpage, video_id, fatal=False, default='{}') video_data.update(traverse_obj(next_data, ('props', 'pageProps', ('title', 'episode')), expected_type=dict)[0] or {}) title = traverse_obj(video_data, 'headerTitle', 'episodeTitle') info = self._og_extract(webpage, require_title=not title) @@ -316,7 +323,7 @@ class ITVIE(ITVBaseIE): class ITVBTCCIE(ITVBaseIE): _VALID_URL = r'https?://(?:www\.)?itv\.com/(?!(?:watch|hub)/)(?:[^/]+/)+(?P[^/?#&]+)' - IE_DESC = 'ITV articles: News, British Touring Car Championship' + _IE_DESC = 'ITV articles: News, British Touring Car Championship' _TESTS = [{ 'note': 'British Touring Car Championship', 'url': 'https://www.itv.com/btcc/articles/btcc-2018-all-the-action-from-brands-hatch', diff --git a/youtube_dl/extractor/senateisvp.py b/youtube_dl/extractor/senateisvp.py index b8ac58713..db5ef8b57 100644 --- a/youtube_dl/extractor/senateisvp.py +++ b/youtube_dl/extractor/senateisvp.py @@ -47,7 +47,7 @@ class SenateISVPIE(InfoExtractor): ['vetaff', '76462', 'http://vetaff-f.akamaihd.net'], ['arch', '', 'http://ussenate-f.akamaihd.net/'] ] - IE_NAME = 'senate.gov' + _IE_NAME = 'senate.gov' _VALID_URL = r'https?://(?:www\.)?senate\.gov/isvp/?\?(?P.+)' _TESTS = [{ 'url': 'http://www.senate.gov/isvp/?comm=judiciary&type=live&stt=&filename=judiciary031715&auto_play=false&wmode=transparent&poster=http%3A%2F%2Fwww.judiciary.senate.gov%2Fthemes%2Fjudiciary%2Fimages%2Fvideo-poster-flash-fit.png', diff --git a/youtube_dl/extractor/youtube.py b/youtube_dl/extractor/youtube.py index b31798729..2e31a8979 100644 --- a/youtube_dl/extractor/youtube.py +++ b/youtube_dl/extractor/youtube.py @@ -3,14 +3,11 @@ from __future__ import unicode_literals import collections -import hashlib import itertools import json import os.path import random import re -import string -import time import traceback from .common import InfoExtractor, SearchInfoExtractor @@ -27,14 +24,11 @@ from ..compat import ( ) from ..jsinterp import JSInterpreter from ..utils import ( - bug_reports_message, clean_html, dict_get, error_to_compat_str, ExtractorError, - filter_dict, float_or_none, - get_first, extract_attributes, get_element_by_attribute, int_or_none, @@ -49,7 +43,6 @@ from ..utils import ( parse_duration, parse_qs, qualities, - remove_end, remove_start, smuggle_url, str_or_none, @@ -67,13 +60,11 @@ from ..utils import ( url_or_none, urlencode_postdata, urljoin, - variadic, ) class YoutubeBaseInfoExtractor(InfoExtractor): """Provide base functions for Youtube extractors""" - _LOGIN_URL = 'https://accounts.google.com/ServiceLogin' _TWOFACTOR_URL = 'https://accounts.google.com/signin/challenge' @@ -87,66 +78,9 @@ class YoutubeBaseInfoExtractor(InfoExtractor): _PLAYLIST_ID_RE = r'(?:(?:PL|LL|EC|UU|FL|RD|UL|TL|PU|OLAK5uy_)[0-9A-Za-z-_]{10,}|RDMM)' - _INNERTUBE_CLIENTS = { - 'ios': { - 'INNERTUBE_CONTEXT': { - 'client': { - 'clientName': 'IOS', - 'clientVersion': '20.10.4', - 'deviceMake': 'Apple', - 'deviceModel': 'iPhone16,2', - 'userAgent': 'com.google.ios.youtube/20.10.4 (iPhone16,2; U; CPU iOS 18_3_2 like Mac OS X;)', - 'osName': 'iPhone', - 'osVersion': '18.3.2.22D82', - }, - }, - 'INNERTUBE_CONTEXT_CLIENT_NAME': 5, - 'REQUIRE_JS_PLAYER': False, - 'REQUIRE_PO_TOKEN': True, - }, - # mweb has 'ultralow' formats - # See: https://github.com/yt-dlp/yt-dlp/pull/557 - 'mweb': { - 'INNERTUBE_CONTEXT': { - 'client': { - 'clientName': 'MWEB', - 'clientVersion': '2.20250311.03.00', - # mweb previously did not require PO Token with this UA - 'userAgent': 'Mozilla/5.0 (iPad; CPU OS 16_7_10 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/16.6 Mobile/15E148 Safari/604.1,gzip(gfe)', - }, - }, - 'INNERTUBE_CONTEXT_CLIENT_NAME': 2, - 'REQUIRE_PO_TOKEN': True, - 'SUPPORTS_COOKIES': True, - }, - 'tv': { - 'INNERTUBE_CONTEXT': { - 'client': { - 'clientName': 'TVHTML5', - 'clientVersion': '7.20250312.16.00', - 'userAgent': 'Mozilla/5.0 (ChromiumStylePlatform) Cobalt/Version', - }, - }, - 'INNERTUBE_CONTEXT_CLIENT_NAME': 7, - 'SUPPORTS_COOKIES': True, - }, - 'web': { - 'INNERTUBE_CONTEXT': { - 'client': { - 'clientName': 'WEB', - 'clientVersion': '2.20250312.04.00', - }, - }, - 'INNERTUBE_CONTEXT_CLIENT_NAME': 1, - 'REQUIRE_PO_TOKEN': True, - 'SUPPORTS_COOKIES': True, - }, - } - def _login(self): """ Attempt to log in to YouTube. - True is returned if successful or skipped. False is returned if login failed. @@ -202,7 +136,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor): [2, 1, None, 1, 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn', None, [], 4], - 1, [None, None, []], None, None, None, True, + 1, [None, None, []], None, None, None, True ], username, ] @@ -224,7 +158,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor): None, 1, None, [1, None, None, None, [password, None, True]], [ None, None, [2, 1, None, 1, 'https://accounts.google.com/ServiceLogin?passive=true&continue=https%3A%2F%2Fwww.youtube.com%2Fsignin%3Fnext%3D%252F%26action_handle_signin%3Dtrue%26hl%3Den%26app%3Ddesktop%26feature%3Dsign_in_button&hl=en&service=youtube&uilel=3&requestPath=%2FServiceLogin&Page=PasswordSeparationSignIn', None, [], 4], - 1, [None, None, []], None, None, None, True, + 1, [None, None, []], None, None, None, True ]] challenge_results = req( @@ -277,7 +211,7 @@ class YoutubeBaseInfoExtractor(InfoExtractor): user_hash, None, 2, None, [ 9, None, None, None, None, None, None, None, - [None, tfa_code, True, 2], + [None, tfa_code, True, 2] ]] tfa_results = req( @@ -343,57 +277,32 @@ class YoutubeBaseInfoExtractor(InfoExtractor): if not self._login(): return - _DEFAULT_API_DATA = {'context': _INNERTUBE_CLIENTS['web']['INNERTUBE_CONTEXT']} + _DEFAULT_API_DATA = { + 'context': { + 'client': { + 'clientName': 'WEB', + 'clientVersion': '2.20201021.03.00', + } + }, + } _YT_INITIAL_DATA_RE = r'(?:window\s*\[\s*["\']ytInitialData["\']\s*\]|ytInitialData)\s*=\s*({.+?})\s*;' _YT_INITIAL_PLAYER_RESPONSE_RE = r'ytInitialPlayerResponse\s*=\s*({.+?})\s*;' _YT_INITIAL_BOUNDARY_RE = r'(?:var\s+meta|[a-zA-Z0-9_-]{8,})/(?:tv-)?player', - r'/(?P[a-zA-Z0-9_-]{8,})/player(?:_ias(?:_tce)?\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$', - r'\b(?Pvfl[a-zA-Z0-9_-]{6,})\b.*?\.js$', + r'/s/player/(?P[a-zA-Z0-9_-]{8,})/player', + r'/(?P[a-zA-Z0-9_-]{8,})/player(?:_ias\.vflset(?:/[a-zA-Z]{2,3}_[a-zA-Z]{2,3})?|-plasma-ias-(?:phone|tablet)-[a-z]{2}_[A-Z]{2}\.vflset)/base\.js$', + r'\b(?Pvfl[a-zA-Z0-9_-]+)\b.*?\.js$', ) _SUBTITLE_FORMATS = ('json3', 'srv1', 'srv2', 'srv3', 'ttml', 'vtt') @@ -773,7 +642,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'uploader_url': r're:https?://(?:www\.)?youtube\.com/user/8KVIDEO', 'description': '', 'uploader': '8KVIDEO', - 'title': 'UHDTV TEST 8K VIDEO.mp4', + 'title': 'UHDTV TEST 8K VIDEO.mp4' }, 'params': { 'youtube_include_dash_manifest': True, @@ -813,7 +682,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'uploader_url': r're:https?://(?:www\.)?youtube\.com/@theamazingatheist', 'title': 'Burning Everyone\'s Koran', 'description': 'SUBSCRIBE: http://www.youtube.com/saturninefilms \r\n\r\nEven Obama has taken a stand against freedom on this issue: http://www.huffingtonpost.com/2010/09/09/obama-gma-interview-quran_n_710282.html', - }, + } }, # Age-gated videos { @@ -941,7 +810,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): }, 'expected_warnings': [ 'DASH manifest missing', - ], + ] }, # Olympics (https://github.com/ytdl-org/youtube-dl/issues/4431) { @@ -1585,15 +1454,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor): '397': {'acodec': 'none', 'vcodec': 'av01.0.05M.08'}, } - _PLAYER_JS_VARIANT_MAP = ( - ('main', 'player_ias.vflset/en_US/base.js'), - ('tce', 'player_ias_tce.vflset/en_US/base.js'), - ('tv', 'tv-player-ias.vflset/tv-player-ias.js'), - ('tv_es6', 'tv-player-es6.vflset/tv-player-es6.js'), - ('phone', 'player-plasma-ias-phone-en_US.vflset/base.js'), - ('tablet', 'player-plasma-ias-tablet-en_US.vflset/base.js'), - ) - @classmethod def suitable(cls, url): if parse_qs(url).get('list', [None])[0]: @@ -1633,97 +1493,46 @@ class YoutubeIE(YoutubeBaseInfoExtractor): """ Return a string representation of a signature """ return '.'.join(compat_str(len(part)) for part in example_sig.split('.')) - def _extract_player_info(self, player_url): - try: - return self._search_regex( - self._PLAYER_INFO_RE, player_url, 'player info', group='id') - except ExtractorError as e: - raise ExtractorError( - 'Cannot identify player %r' % (player_url,), cause=e) + @classmethod + def _extract_player_info(cls, player_url): + for player_re in cls._PLAYER_INFO_RE: + id_m = re.search(player_re, player_url) + if id_m: + break + else: + raise ExtractorError('Cannot identify player %r' % player_url) + return id_m.group('id') - def _player_js_cache_key(self, player_url, extra_id=None, _cache={}): - if player_url not in _cache: + def _load_player(self, video_id, player_url, fatal=True, player_id=None): + if not player_id: player_id = self._extract_player_info(player_url) - player_path = remove_start( - compat_urllib_parse.urlparse(player_url).path, - '/s/player/{0}/'.format(player_id)) - variant = next((k for k, v in self._PLAYER_JS_VARIANT_MAP - if v == player_path), None) - if not variant: - variant = next( - (k for k, v in self._PLAYER_JS_VARIANT_MAP - if re.match(re.escape(v).replace('en_US', r'\w+') + '$', player_path)), - None) - if not variant: - self.write_debug( - 'Unable to determine player JS variant\n' - ' player = {0}'.format(player_url), only_once=True) - variant = re.sub(r'[^a-zA-Z0-9]', '_', remove_end(player_path, '.js')) - _cache[player_url] = join_nonempty(player_id, variant) - - if extra_id: - extra_id = '-'.join((_cache[player_url], extra_id)) - assert os.path.basename(extra_id) == extra_id - return extra_id - return _cache[player_url] - - def _load_player(self, video_id, player_url, fatal=True): - player_js_key = self._player_js_cache_key(player_url) - if player_js_key not in self._code_cache: + if player_id not in self._code_cache: code = self._download_webpage( player_url, video_id, fatal=fatal, - note='Downloading player {0}'.format(player_js_key), - errnote='Download of {0} failed'.format(player_url)) + note='Downloading player ' + player_id, + errnote='Download of %s failed' % player_url) if code: - self._code_cache[player_js_key] = code - return self._code_cache.get(player_js_key) - - def _load_player_data_from_cache(self, name, player_url, extra_id=None): - cache_id = ('youtube-{0}'.format(name), self._player_js_cache_key(player_url, extra_id)) - data = self._player_cache.get(cache_id) - if data: - return data - - data = self.cache.load(*cache_id, min_ver='2025.04.07') - if data: - self._player_cache[cache_id] = data - return data - - def _store_player_data_to_cache(self, name, player_url, data, extra_id=None): - cache_id = ('youtube-{0}'.format(name), self._player_js_cache_key(player_url, extra_id)) - - if cache_id not in self._player_cache: - self.cache.store(cache_id[0], cache_id[1], data) - self._player_cache[cache_id] = data - - def _remove_player_data_from_cache(self, name, player_url, extra_id=None): - cache_id = ('youtube-{0}'.format(name), self._player_js_cache_key(player_url, extra_id)) - - if cache_id in self._player_cache: - self.cache.clear(*cache_id) - self._player_cache.pop(cache_id, None) + self._code_cache[player_id] = code + return self._code_cache[player_id] if fatal else self._code_cache.get(player_id) def _extract_signature_function(self, video_id, player_url, example_sig): - # player_id = self._extract_player_info(player_url) + player_id = self._extract_player_info(player_url) # Read from filesystem cache - extra_id = self._signature_cache_id(example_sig) - self.write_debug('Extracting signature function {0}-{1}'.format(player_url, extra_id)) - cache_spec, code = self._load_player_data_from_cache( - 'sigfuncs', player_url, extra_id=extra_id), None + func_id = 'js_{0}_{1}'.format( + player_id, self._signature_cache_id(example_sig)) + assert os.path.basename(func_id) == func_id + + self.write_debug('Extracting signature function {0}'.format(func_id)) + cache_spec, code = self.cache.load('youtube-sigfuncs', func_id), None if not cache_spec: - code = self._load_player(video_id, player_url) - if code: - res = self._parse_sig_js(code) - test_string = ''.join(map(compat_chr, range(len(example_sig)))) - cache_spec = [ord(c) for c in res(test_string)] - self._store_player_data_to_cache( - 'sigfuncs', player_url, cache_spec, extra_id=extra_id) - else: - self.report_warning( - 'Failed to compute signature function {0}-{1}'.format( - player_url, extra_id)) + code = self._load_player(video_id, player_url, player_id) + if code: + res = self._parse_sig_js(code) + test_string = ''.join(map(compat_chr, range(len(example_sig)))) + cache_spec = [ord(c) for c in res(test_string)] + self.cache.store('youtube-sigfuncs', func_id, cache_spec) return lambda s: ''.join(s[i] for i in cache_spec) @@ -1769,54 +1578,26 @@ class YoutubeIE(YoutubeBaseInfoExtractor): ' return %s\n') % (signature_id_tuple, expr_code) self.to_screen('Extracted signature function:\n' + code) - def _extract_sig_fn(self, jsi, funcname): - var_ay = self._search_regex( - r'''(?x) - (?:\*/|\{|\n|^)\s*(?:'[^']+'\s*;\s*) - (var\s*[\w$]+\s*=\s*(?: - ('|")(?:\\\2|(?!\2).)+\2\s*\.\s*split\(\s*('|")\W+\3\s*\)| - \[\s*(?:('|")(?:\\\4|(?!\4).)*\4\s*(?:(?=\])|,\s*))+\] - ))(?=\s*[,;]) - ''', jsi.code, 'useful values', default='') - - sig_fn = jsi.extract_function_code(funcname) - - if var_ay: - sig_fn = (sig_fn[0], ';\n'.join((var_ay, sig_fn[1]))) - - return sig_fn - def _parse_sig_js(self, jscode): - # Examples where `sig` is funcname: - # sig=function(a){a=a.split(""); ... ;return a.join("")}; - # ;c&&(c=sig(decodeURIComponent(c)),a.set(b,encodeURIComponent(c)));return a}; - # {var l=f,m=h.sp,n=sig(decodeURIComponent(h.s));l.set(m,encodeURIComponent(n))} - # sig=function(J){J=J.split(""); ... ;return J.join("")}; - # ;N&&(N=sig(decodeURIComponent(N)),J.set(R,encodeURIComponent(N)));return J}; - # {var H=u,k=f.sp,v=sig(decodeURIComponent(f.s));H.set(k,encodeURIComponent(v))} funcname = self._search_regex( - (r'\b(?P[\w$]+)&&\((?P=var)=(?P[\w$]{2,})\(decodeURIComponent\((?P=var)\)\)', - r'(?P[\w$]+)\s*=\s*function\(\s*(?P[\w$]+)\s*\)\s*{\s*(?P=arg)\s*=\s*(?P=arg)\.split\(\s*""\s*\)\s*;\s*[^}]+;\s*return\s+(?P=arg)\.join\(\s*""\s*\)', - r'(?:\b|[^\w$])(?P[\w$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[\w$]{2}\.[\w$]{2}\(a,\d+\))?', - # Old patterns - r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P[\w$]+)\(', - r'\b[\w]+\s*&&\s*[\w]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P[\w$]+)\(', - r'\bm=(?P[\w$]{2,})\(decodeURIComponent\(h\.s\)\)', + (r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P[a-zA-Z0-9$]+)\(', + r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*encodeURIComponent\s*\(\s*(?P[a-zA-Z0-9$]+)\(', + r'\bm=(?P[a-zA-Z0-9$]{2,})\(decodeURIComponent\(h\.s\)\)', + r'\bc&&\(c=(?P[a-zA-Z0-9$]{2,})\(decodeURIComponent\(c\)\)', + r'(?:\b|[^a-zA-Z0-9$])(?P[a-zA-Z0-9$]{2,})\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)(?:;[a-zA-Z0-9$]{2}\.[a-zA-Z0-9$]{2}\(a,\d+\))?', + r'(?P[a-zA-Z0-9$]+)\s*=\s*function\(\s*a\s*\)\s*{\s*a\s*=\s*a\.split\(\s*""\s*\)', # Obsolete patterns - r'("|\')signature\1\s*,\s*(?P[\w$]+)\(', - r'\.sig\|\|(?P[\w$]+)\(', - r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P[\w$]+)\(', - r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P[\w$]+)\(', - r'\bc\s*&&\s*[\w]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P[\w$]+)\('), + r'("|\')signature\1\s*,\s*(?P[a-zA-Z0-9$]+)\(', + r'\.sig\|\|(?P[a-zA-Z0-9$]+)\(', + r'yt\.akamaized\.net/\)\s*\|\|\s*.*?\s*[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?:encodeURIComponent\s*\()?\s*(?P[a-zA-Z0-9$]+)\(', + r'\b[cs]\s*&&\s*[adf]\.set\([^,]+\s*,\s*(?P[a-zA-Z0-9$]+)\(', + r'\b[a-zA-Z0-9]+\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*(?P[a-zA-Z0-9$]+)\(', + r'\bc\s*&&\s*[a-zA-Z0-9]+\.set\([^,]+\s*,\s*\([^)]*\)\s*\(\s*(?P[a-zA-Z0-9$]+)\('), jscode, 'Initial JS player signature function name', group='sig') jsi = JSInterpreter(jscode) - - initial_function = self._extract_sig_fn(jsi, funcname) - - func = jsi.extract_function_from_code(*initial_function) - - return lambda s: func([s]) + initial_function = jsi.extract_function(funcname) + return lambda s: initial_function([s]) def _cached(self, func, *cache_id): def inner(*args, **kwargs): @@ -1866,7 +1647,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): except JSInterpreter.Exception as e: self.report_warning( '%s (%s %s)' % ( - 'Unable to decode n-parameter: expect download to be blocked or throttled', + 'Unable to decode n-parameter: download likely to be throttled', error_to_compat_str(e), traceback.format_exc()), video_id=video_id) @@ -1876,103 +1657,37 @@ class YoutubeIE(YoutubeBaseInfoExtractor): return ret def _extract_n_function_name(self, jscode): - func_name, idx = None, None - - def generic_n_function_search(func_name=None): - return self._search_regex( - r'''(?xs) - (?:(?<=[^\w$])|^) # instead of \b, which ignores $ - (?P%s)\s*=\s*function\((?!\d)[a-zA-Z\d_$]+\) - \s*\{(?:(?!};).)+?(?: - ["']enhanced_except_ | - return\s*(?P"|')[a-zA-Z\d-]+_w8_(?P=q)\s*\+\s*[\w$]+ - ) - ''' % (func_name or r'(?!\d)[a-zA-Z\d_$]+',), jscode, - 'Initial JS player n function name', group='name', - default=None if func_name else NO_DEFAULT) - - # these special cases are redundant and probably obsolete (2025-04): - # they make the tests run ~10% faster without fallback warnings - r""" func_name, idx = self._search_regex( - # (y=NuD(),Mw(k),q=k.Z[y]||null)&&(q=narray[idx](q),k.set(y,q),k.V||NuD(''))}}; - # (R="nn"[+J.Z],mW(J),N=J.K[R]||null)&&(N=narray[idx](N),J.set(R,N))}}; - # or: (b=String.fromCharCode(110),c=a.get(b))&&c=narray[idx](c) - # or: (b="nn"[+a.D],c=a.get(b))&&(c=narray[idx](c) - # or: (PL(a),b=a.j.n||null)&&(b=narray[idx](b) - # or: (b="nn"[+a.D],vL(a),c=a.j[b]||null)&&(c=narray[idx](c),a.set(b,c),narray.length||nfunc("") - # old: (b=a.get("n"))&&(b=narray[idx](b)(?P[a-z])\s*=\s*[a-z]\s* - # older: (b=a.get("n"))&&(b=nfunc(b) + # new: (b=String.fromCharCode(110),c=a.get(b))&&c=nfunc[idx](c) + # old: .get("n"))&&(b=nfunc[idx](b) + # older: .get("n"))&&(b=nfunc(b) r'''(?x) - # (expr, ..., - \((?:(?:\s*[\w$]+\s*=)?(?:[\w$"+\.\s(\[]+(?:[)\]]\s*)?),)* - # b=... - (?P[\w$]+)\s*=\s*(?!(?P=b)[^\w$])[\w$]+\s*(?:(?: - \.\s*[\w$]+ | - \[\s*[\w$]+\s*\] | - \.\s*get\s*\(\s*[\w$"]+\s*\) - )\s*){,2}(?:\s*\|\|\s*null(?=\s*\)))?\s* - \)\s*&&\s*\( # ...)&&( - # b = nfunc, b = narray[idx] - (?P=b)\s*=\s*(?P[\w$]+)\s* - (?:\[\s*(?P[\w$]+)\s*\]\s*)? - # (...) - \(\s*[\w$]+\s*\) - ''', jscode, 'Initial JS player n function name', group=('nfunc', 'idx'), - default=(None, None)) - """ - - if not func_name: - # nfunc=function(x){...}|function nfunc(x); ... - # ... var y=[nfunc]|y[idx]=nfunc); - # obvious REs hang, so use a two-stage tactic - for m in re.finditer(r'''(?x) - [\n;]var\s(?:(?:(?!,).)+,|\s)*?(?!\d)[\w$]+(?:\[(?P\d+)\])?\s*=\s* - (?(idx)|\[\s*)(?P(?!\d)[\w$]+)(?(idx)|\s*\]) - \s*?[;\n] - ''', jscode): - fn = self._search_regex( - r'[;,]\s*(function\s+)?({0})(?(1)|\s*=\s*function)\s*\((?!\d)[\w$]+\)\s*\{1}(?!\s*return\s)'.format( - re.escape(m.group('nfunc')), '{'), - jscode, 'Initial JS player n function name (2)', group=2, default=None) - if fn: - func_name = fn - idx = m.group('idx') - if generic_n_function_search(func_name): - # don't look any further - break - - # thx bashonly: yt-dlp/yt-dlp/pull/10611 - if not func_name: - self.report_warning('Falling back to generic n function search', only_once=True) - return generic_n_function_search() - + (?:\(\s*(?P[a-z])\s*=\s*String\s*\.\s*fromCharCode\s*\(\s*110\s*\)\s*,(?P[a-z])\s*=\s*[a-z]\s*)? + \.\s*get\s*\(\s*(?(b)(?P=b)|"n")(?:\s*\)){2}\s*&&\s*\(\s*(?(c)(?P=c)|b)\s*=\s* + (?P[a-zA-Z_$][\w$]*)(?:\s*\[(?P\d+)\])?\s*\(\s*[\w$]+\s*\) + ''', jscode, 'Initial JS player n function name', group=('nfunc', 'idx')) if not idx: return func_name - return self._search_json( - r'(?[0-9]{5})', code or '', - 'JS player signature timestamp', group='sts', fatal=fatal)) - if sts: - self._store_player_data_to_cache('sts', player_url, sts) - + if not sts: + # Attempt to extract from player + if player_url is None: + error_msg = 'Cannot extract signature timestamp without player_url.' + if fatal: + raise ExtractorError(error_msg) + self.report_warning(error_msg) + return + code = self._load_player(video_id, player_url, fatal=fatal) + sts = int_or_none(self._search_regex( + r'(?:signatureTimestamp|sts)\s*:\s*(?P[0-9]{5})', code or '', + 'JS player signature timestamp', group='sts', fatal=fatal)) return sts def _mark_watched(self, video_id, player_response): @@ -2050,8 +1754,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor): # cpn generation algorithm is reverse engineered from base.js. # In fact it works even with dummy cpn. - CPN_ALPHABET = string.ascii_letters + string.digits + '-_' - cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(16)) + CPN_ALPHABET = 'abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789-_' + cpn = ''.join(CPN_ALPHABET[random.randint(0, 256) & 63] for _ in range(0, 16)) # more consistent results setting it to right before the end qs = parse_qs(playback_url) @@ -2111,7 +1815,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor): mobj = re.match(cls._VALID_URL, url, re.VERBOSE) if mobj is None: raise ExtractorError('Invalid URL: %s' % url) - return mobj.group(2) + video_id = mobj.group(2) + return video_id def _extract_chapters_from_json(self, data, video_id, duration): chapters_list = try_get( @@ -2172,89 +1877,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor): player_response = self._extract_yt_initial_variable( webpage, self._YT_INITIAL_PLAYER_RESPONSE_RE, video_id, 'initial player response') - is_live = traverse_obj(player_response, ('videoDetails', 'isLive')) - - if False and not player_response: + if not player_response: player_response = self._call_api( 'player', {'videoId': video_id}, video_id) - if True or not player_response: - origin = 'https://www.youtube.com' - pb_context = {'html5Preference': 'HTML5_PREF_WANTS'} - - player_url = self._extract_player_url(webpage) - ytcfg = self._extract_ytcfg(video_id, webpage or '') - sts = self._extract_signature_timestamp(video_id, player_url, ytcfg) - if sts: - pb_context['signatureTimestamp'] = sts - - client_names = traverse_obj(self._INNERTUBE_CLIENTS, ( - T(dict.items), lambda _, k_v: not k_v[1].get('REQUIRE_PO_TOKEN'), - 0))[:1] - if 'web' not in client_names: - # webpage links won't download: ignore links and playability - player_response = filter_dict( - player_response or {}, - lambda k, _: k not in ('streamingData', 'playabilityStatus')) - - if is_live and 'ios' not in client_names: - client_names.append('ios') - - headers = { - 'Sec-Fetch-Mode': 'navigate', - 'Origin': origin, - 'X-Goog-Visitor-Id': self._extract_visitor_data(ytcfg) or '', - } - auth = self._generate_sapisidhash_header(origin) - if auth is not None: - headers['Authorization'] = auth - headers['X-Origin'] = origin - - for client in traverse_obj(self._INNERTUBE_CLIENTS, (client_names, T(dict))): - - query = { - 'playbackContext': { - 'contentPlaybackContext': pb_context, - }, - 'contentCheckOk': True, - 'racyCheckOk': True, - 'context': { - 'client': merge_dicts( - traverse_obj(client, ('INNERTUBE_CONTEXT', 'client')), { - 'hl': 'en', - 'timeZone': 'UTC', - 'utcOffsetMinutes': 0, - }), - }, - 'videoId': video_id, - } - - api_headers = merge_dicts(headers, traverse_obj(client, { - 'X-YouTube-Client-Name': 'INNERTUBE_CONTEXT_CLIENT_NAME', - 'X-YouTube-Client-Version': ( - 'INNERTUBE_CONTEXT', 'client', 'clientVersion'), - 'User-Agent': ( - 'INNERTUBE_CONTEXT', 'client', 'userAgent'), - })) - - api_player_response = self._call_api( - 'player', query, video_id, fatal=False, headers=api_headers, - note=join_nonempty( - 'Downloading', traverse_obj(query, ( - 'context', 'client', 'clientName')), - 'API JSON', delim=' ')) - - hls = traverse_obj( - (player_response, api_player_response), - (Ellipsis, 'streamingData', 'hlsManifestUrl', T(url_or_none))) - if len(hls) == 2 and not hls[0] and hls[1]: - player_response['streamingData']['hlsManifestUrl'] = hls[1] - else: - video_details = merge_dicts(*traverse_obj( - (player_response, api_player_response), - (Ellipsis, 'videoDetails', T(dict)))) - player_response.update(filter_dict( - api_player_response or {}, cndn=lambda k, _: k != 'captions')) - player_response['videoDetails'] = video_details def is_agegated(playability): if not isinstance(playability, dict): @@ -2303,7 +1928,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): headers = { 'X-YouTube-Client-Name': '85', 'X-YouTube-Client-Version': '2.0', - 'Origin': 'https://www.youtube.com', + 'Origin': 'https://www.youtube.com' } video_info = self._call_api('player', query, video_id, fatal=False, headers=headers) @@ -2332,8 +1957,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor): return ''.join([r['text'] for r in runs if isinstance(r.get('text'), compat_str)]) search_meta = ( - (lambda x: self._html_search_meta(x, webpage, default=None)) - if webpage else lambda _: None) + lambda x: self._html_search_meta(x, webpage, default=None)) \ + if webpage else lambda x: None video_details = player_response.get('videoDetails') or {} microformat = try_get( @@ -2397,7 +2022,6 @@ class YoutubeIE(YoutubeBaseInfoExtractor): itag_qualities = {} q = qualities(['tiny', 'small', 'medium', 'large', 'hd720', 'hd1080', 'hd1440', 'hd2160', 'hd2880', 'highres']) CHUNK_SIZE = 10 << 20 - is_live = video_details.get('isLive') streaming_data = player_response.get('streamingData') or {} streaming_formats = streaming_data.get('formats') or [] @@ -2406,7 +2030,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): def build_fragments(f): return LazyList({ 'url': update_url_query(f['url'], { - 'range': '{0}-{1}'.format(range_start, min(range_start + CHUNK_SIZE - 1, f['filesize'])), + 'range': '{0}-{1}'.format(range_start, min(range_start + CHUNK_SIZE - 1, f['filesize'])) }) } for range_start in range(0, f['filesize'], CHUNK_SIZE)) @@ -2505,7 +2129,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'protocol': 'http_dash_segments', 'fragments': build_fragments(dct), } if dct['filesize'] else { - 'downloader_options': {'http_chunk_size': CHUNK_SIZE}, # No longer useful? + 'downloader_options': {'http_chunk_size': CHUNK_SIZE} # No longer useful? }) formats.append(dct) @@ -2542,8 +2166,7 @@ class YoutubeIE(YoutubeBaseInfoExtractor): hls_manifest_url = streaming_data.get('hlsManifestUrl') if hls_manifest_url: for f in self._extract_m3u8_formats( - hls_manifest_url, video_id, 'mp4', - entry_protocol='m3u8_native', live=is_live, fatal=False): + hls_manifest_url, video_id, 'mp4', fatal=False): if process_manifest_format( f, 'hls', None, self._search_regex( r'/itag/(\d+)', f['url'], 'itag', default=None)): @@ -2563,12 +2186,12 @@ class YoutubeIE(YoutubeBaseInfoExtractor): formats.append(f) playable_formats = [f for f in formats if not f.get('has_drm')] - if formats: - if not playable_formats: - # If there are no formats that definitely don't have DRM, all have DRM - self.report_drm(video_id) - formats[:] = playable_formats - else: + if formats and not playable_formats: + # If there are no formats that definitely don't have DRM, all have DRM + self.report_drm(video_id) + formats[:] = playable_formats + + if not formats: if streaming_data.get('licenseInfos'): raise ExtractorError( 'This video is DRM protected.', expected=True) @@ -2649,6 +2272,8 @@ class YoutubeIE(YoutubeBaseInfoExtractor): # Strictly de-prioritize damaged formats f['preference'] = -10 + is_live = video_details.get('isLive') + owner_profile_url = self._yt_urljoin(self._extract_author_var( webpage, 'url', videodetails=video_details, metadata=microformat)) @@ -2682,9 +2307,9 @@ class YoutubeIE(YoutubeBaseInfoExtractor): 'is_live': is_live, } - pctr = traverse_obj( - (player_response, api_player_response), - (Ellipsis, 'captions', 'playerCaptionsTracklistRenderer', T(dict))) + pctr = try_get( + player_response, + lambda x: x['captions']['playerCaptionsTracklistRenderer'], dict) if pctr: def process_language(container, base_url, lang_code, query): lang_subs = [] @@ -2698,35 +2323,31 @@ class YoutubeIE(YoutubeBaseInfoExtractor): }) container[lang_code] = lang_subs - def process_subtitles(): - subtitles = {} - for caption_track in traverse_obj(pctr, ( - Ellipsis, 'captionTracks', lambda _, v: ( - v.get('baseUrl') and v.get('languageCode')))): - base_url = self._yt_urljoin(caption_track['baseUrl']) - if not base_url: + subtitles = {} + for caption_track in (pctr.get('captionTracks') or []): + base_url = caption_track.get('baseUrl') + if not base_url: + continue + if caption_track.get('kind') != 'asr': + lang_code = caption_track.get('languageCode') + if not lang_code: continue - lang_code = caption_track['languageCode'] - if caption_track.get('kind') != 'asr': - process_language( - subtitles, base_url, lang_code, {}) - continue - automatic_captions = {} process_language( - automatic_captions, base_url, lang_code, {}) - for translation_language in traverse_obj(pctr, ( - Ellipsis, 'translationLanguages', lambda _, v: v.get('languageCode'))): - translation_language_code = translation_language['languageCode'] - process_language( - automatic_captions, base_url, translation_language_code, - {'tlang': translation_language_code}) - info['automatic_captions'] = automatic_captions - info['subtitles'] = subtitles - - process_subtitles() + subtitles, base_url, lang_code, {}) + continue + automatic_captions = {} + for translation_language in (pctr.get('translationLanguages') or []): + translation_language_code = translation_language.get('languageCode') + if not translation_language_code: + continue + process_language( + automatic_captions, base_url, translation_language_code, + {'tlang': translation_language_code}) + info['automatic_captions'] = automatic_captions + info['subtitles'] = subtitles parsed_url = compat_urllib_parse_urlparse(url) - for component in (parsed_url.fragment, parsed_url.query): + for component in [parsed_url.fragment, parsed_url.query]: query = compat_parse_qs(component) for k, v in query.items(): for d_k, s_ks in [('start', ('start', 't')), ('end', ('end',))]: @@ -2956,7 +2577,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): 'title': 'Super Cooper Shorts - Shorts', 'uploader': 'Super Cooper Shorts', 'uploader_id': '@SuperCooperShorts', - }, + } }, { # Channel that does not have a Shorts tab. Test should just download videos on Home tab instead 'url': 'https://www.youtube.com/@emergencyawesome/shorts', @@ -3010,7 +2631,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): 'description': 'md5:609399d937ea957b0f53cbffb747a14c', 'uploader': 'ThirstForScience', 'uploader_id': '@ThirstForScience', - }, + } }, { 'url': 'https://www.youtube.com/c/ChristophLaimer/playlists', 'only_matching': True, @@ -3309,7 +2930,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): 'uploader': '3Blue1Brown', 'uploader_id': '@3blue1brown', 'channel_id': 'UCYO_jab_esuFRV4b17AJtAw', - }, + } }] @classmethod @@ -3334,12 +2955,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): expected_type=txt_or_none) def _grid_entries(self, grid_renderer): - for item in traverse_obj(grid_renderer, ('items', Ellipsis, T(dict))): - lockup_view_model = traverse_obj(item, ('lockupViewModel', T(dict))) - if lockup_view_model: - entry = self._extract_lockup_view_model(lockup_view_model) - if entry: - yield entry + for item in grid_renderer['items']: + if not isinstance(item, dict): continue renderer = self._extract_grid_item_renderer(item) if not isinstance(renderer, dict): @@ -3423,39 +3040,6 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): continue yield self._extract_video(renderer) - def _extract_lockup_view_model(self, view_model): - content_id = view_model.get('contentId') - if not content_id: - return - content_type = view_model.get('contentType') - if content_type not in ('LOCKUP_CONTENT_TYPE_PLAYLIST', 'LOCKUP_CONTENT_TYPE_PODCAST'): - self.report_warning( - 'Unsupported lockup view model content type "{0}"{1}'.format(content_type, bug_reports_message()), only_once=True) - return - return merge_dicts(self.url_result( - update_url_query('https://www.youtube.com/playlist', {'list': content_id}), - ie=YoutubeTabIE.ie_key(), video_id=content_id), { - 'title': traverse_obj(view_model, ( - 'metadata', 'lockupMetadataViewModel', 'title', 'content', T(compat_str))), - 'thumbnails': self._extract_thumbnails(view_model, ( - 'contentImage', 'collectionThumbnailViewModel', 'primaryThumbnail', - 'thumbnailViewModel', 'image'), final_key='sources'), - }) - - def _extract_shorts_lockup_view_model(self, view_model): - content_id = traverse_obj(view_model, ( - 'onTap', 'innertubeCommand', 'reelWatchEndpoint', 'videoId', - T(lambda v: v if YoutubeIE.suitable(v) else None))) - if not content_id: - return - return merge_dicts(self.url_result( - content_id, ie=YoutubeIE.ie_key(), video_id=content_id), { - 'title': traverse_obj(view_model, ( - 'overlayMetadata', 'primaryText', 'content', T(compat_str))), - 'thumbnails': self._extract_thumbnails( - view_model, 'thumbnail', final_key='sources'), - }) - def _video_entry(self, video_renderer): video_id = video_renderer.get('videoId') if video_id: @@ -3502,9 +3086,10 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): yield entry def _rich_grid_entries(self, contents): - for content in traverse_obj( - contents, (Ellipsis, 'richItemRenderer', 'content'), - expected_type=dict): + for content in contents: + content = traverse_obj( + content, ('richItemRenderer', 'content'), + expected_type=dict) or {} video_renderer = traverse_obj( content, 'videoRenderer', 'reelItemRenderer', expected_type=dict) @@ -3512,12 +3097,6 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): entry = self._video_entry(video_renderer) if entry: yield entry - # shorts item - shorts_lockup_view_model = content.get('shortsLockupViewModel') - if shorts_lockup_view_model: - entry = self._extract_shorts_lockup_view_model(shorts_lockup_view_model) - if entry: - yield entry # playlist renderer = traverse_obj( content, 'playlistRenderer', expected_type=dict) or {} @@ -3556,15 +3135,23 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): next_continuation = cls._extract_next_continuation_data(renderer) if next_continuation: return next_continuation - for command in traverse_obj(renderer, ( - ('contents', 'items', 'rows'), Ellipsis, 'continuationItemRenderer', - ('continuationEndpoint', ('button', 'buttonRenderer', 'command')), - (('commandExecutorCommand', 'commands', Ellipsis), None), T(dict))): - continuation = traverse_obj(command, ('continuationCommand', 'token', T(compat_str))) + contents = [] + for key in ('contents', 'items'): + contents.extend(try_get(renderer, lambda x: x[key], list) or []) + for content in contents: + if not isinstance(content, dict): + continue + continuation_ep = try_get( + content, lambda x: x['continuationItemRenderer']['continuationEndpoint'], + dict) + if not continuation_ep: + continue + continuation = try_get( + continuation_ep, lambda x: x['continuationCommand']['token'], compat_str) if not continuation: continue - ctp = command.get('clickTrackingParams') - return cls._build_continuation_query(continuation, ctp) + ctp = continuation_ep.get('clickTrackingParams') + return YoutubeTabIE._build_continuation_query(continuation, ctp) def _entries(self, tab, item_id, webpage): tab_content = try_get(tab, lambda x: x['content'], dict) @@ -3613,13 +3200,6 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): entry = self._video_entry(renderer) if entry: yield entry - renderer = isr_content.get('richGridRenderer') - if renderer: - for from_ in self._rich_grid_entries( - traverse_obj(renderer, ('contents', Ellipsis, T(dict)))): - yield from_ - continuation = self._extract_continuation(renderer) - continue if not continuation: continuation = self._extract_continuation(is_renderer) @@ -3629,9 +3209,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): rich_grid_renderer = tab_content.get('richGridRenderer') if not rich_grid_renderer: return - for from_ in self._rich_grid_entries( - traverse_obj(rich_grid_renderer, ('contents', Ellipsis, T(dict)))): - yield from_ + for entry in self._rich_grid_entries(rich_grid_renderer.get('contents') or []): + yield entry continuation = self._extract_continuation(rich_grid_renderer) @@ -3649,7 +3228,7 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): 'client': { 'clientName': 'WEB', 'clientVersion': client_version, - }, + } } visitor_data = try_get(context, lambda x: x['client']['visitorData'], compat_str) @@ -3665,10 +3244,10 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): if not continuation: break if visitor_data: - headers['X-Goog-Visitor-Id'] = visitor_data + headers['x-goog-visitor-id'] = visitor_data data['continuation'] = continuation['continuation'] data['clickTracking'] = { - 'clickTrackingParams': continuation['itct'], + 'clickTrackingParams': continuation['itct'] } count = 0 retries = 3 @@ -3677,12 +3256,8 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): # Downloading page may result in intermittent 5xx HTTP error # that is usually worked around with a retry response = self._download_json( - 'https://www.youtube.com/youtubei/v1/browse', + 'https://www.youtube.com/youtubei/v1/browse?key=AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8', None, 'Downloading page %d%s' % (page_num, ' (retry #%d)' % count if count else ''), - query={ - # 'key': 'AIzaSyAO_FJ2SlqU8Q4STEHLGCilw_Y9_11qcW8', - 'prettyPrint': 'false', - }, headers=headers, data=json.dumps(data).encode('utf8')) break except ExtractorError as e: @@ -3851,23 +3426,10 @@ class YoutubeTabIE(YoutubeBaseInfoExtractor): def _real_extract(self, url): item_id = self._match_id(url) url = update_url(url, netloc='www.youtube.com') - qs = parse_qs(url) - - def qs_get(key, default=None): - return qs.get(key, [default])[-1] - - # Go around for /feeds/videos.xml?playlist_id={pl_id} - if item_id == 'feeds' and '/feeds/videos.xml?' in url: - playlist_id = qs_get('playlist_id') - if playlist_id: - return self.url_result( - update_url_query('https://www.youtube.com/playlist', { - 'list': playlist_id, - }), ie=self.ie_key(), video_id=playlist_id) - # Handle both video/playlist URLs - video_id = qs_get('v') - playlist_id = qs_get('list') + qs = parse_qs(url) + video_id = qs.get('v', [None])[0] + playlist_id = qs.get('list', [None])[0] if video_id and playlist_id: if self._downloader.params.get('noplaylist'): self.to_screen('Downloading just video %s because of --no-playlist' % video_id) @@ -3944,7 +3506,7 @@ class YoutubePlaylistIE(InfoExtractor): 'uploader': 'milan', 'uploader_id': '@milan5503', 'channel_id': 'UCEI1-PVPcYXjB73Hfelbmaw', - }, + } }, { 'url': 'http://www.youtube.com/embed/_xDOZElKyNU?list=PLsyOSbh5bs16vubvKePAQ1x3PhKavfBIl', 'playlist_mincount': 455, @@ -3954,7 +3516,7 @@ class YoutubePlaylistIE(InfoExtractor): 'uploader': 'LBK', 'uploader_id': '@music_king', 'channel_id': 'UC21nz3_MesPLqtDqwdvnoxA', - }, + } }, { 'url': 'TLGGrESM50VT6acwMjAyMjAxNw', 'only_matching': True, @@ -4065,7 +3627,7 @@ class YoutubeSearchIE(SearchInfoExtractor, YoutubeBaseInfoExtractor): 'info_dict': { 'id': 'youtube-dl test video', 'title': 'youtube-dl test video', - }, + } }] def _get_n_results(self, query, n): @@ -4085,7 +3647,7 @@ class YoutubeSearchDateIE(YoutubeSearchIE): 'info_dict': { 'id': 'youtube-dl test video', 'title': 'youtube-dl test video', - }, + } }] @@ -4100,7 +3662,7 @@ class YoutubeSearchURLIE(YoutubeBaseInfoExtractor): 'id': 'youtube-dl test video', 'title': 'youtube-dl test video', }, - 'params': {'playlistend': 5}, + 'params': {'playlistend': 5} }, { 'url': 'https://www.youtube.com/results?q=test&sp=EgQIBBgB', 'only_matching': True, @@ -4116,7 +3678,6 @@ class YoutubeSearchURLIE(YoutubeBaseInfoExtractor): class YoutubeFeedsInfoExtractor(YoutubeTabIE): """ Base class for feed extractors - Subclasses must define the _FEED_NAME property. """ _LOGIN_REQUIRED = True diff --git a/youtube_dl/jsinterp.py b/youtube_dl/jsinterp.py index 7630e2099..949f77775 100644 --- a/youtube_dl/jsinterp.py +++ b/youtube_dl/jsinterp.py @@ -1,23 +1,17 @@ -# coding: utf-8 from __future__ import unicode_literals -import calendar import itertools import json import operator import re -import time -from functools import update_wrapper, wraps +from functools import update_wrapper from .utils import ( error_to_compat_str, ExtractorError, - float_or_none, - int_or_none, js_to_json, remove_quotes, - str_or_none, unified_timestamp, variadic, write_string, @@ -26,13 +20,9 @@ from .compat import ( compat_basestring, compat_chr, compat_collections_chain_map as ChainMap, - compat_contextlib_suppress, compat_filter as filter, - compat_int, - compat_integer_types, compat_itertools_zip_longest as zip_longest, compat_map as map, - compat_numeric_types, compat_str, ) @@ -72,144 +62,55 @@ _NaN = float('nan') _Infinity = float('inf') -class JS_Undefined(object): - pass +def _js_bit_op(op): - -def _js_bit_op(op, is_shift=False): - - def zeroise(x, is_shift_arg=False): - if isinstance(x, compat_integer_types): - return (x % 32) if is_shift_arg else (x & 0xffffffff) - try: - x = float(x) - if is_shift_arg: - x = int(x % 32) - elif x < 0: - x = -compat_int(-x % 0xffffffff) - else: - x = compat_int(x % 0xffffffff) - except (ValueError, TypeError): - # also here for int(NaN), including float('inf') % 32 - x = 0 - return x + def zeroise(x): + return 0 if x in (None, JS_Undefined, _NaN, _Infinity) else x @wraps_op(op) def wrapped(a, b): - return op(zeroise(a), zeroise(b, is_shift)) & 0xffffffff + return op(zeroise(a), zeroise(b)) & 0xffffffff return wrapped -def _js_arith_op(op, div=False): +def _js_arith_op(op): @wraps_op(op) def wrapped(a, b): if JS_Undefined in (a, b): return _NaN - # null, "" --> 0 - a, b = (float_or_none( - (x.strip() if isinstance(x, compat_basestring) else x) or 0, - default=_NaN) for x in (a, b)) - if _NaN in (a, b): - return _NaN - try: - return op(a, b) - except ZeroDivisionError: - return _NaN if not (div and (a or b)) else _Infinity + return op(a or 0, b or 0) return wrapped -_js_arith_add = _js_arith_op(operator.add) +def _js_div(a, b): + if JS_Undefined in (a, b) or not (a or b): + return _NaN + return operator.truediv(a or 0, b) if b else _Infinity -def _js_add(a, b): - if not (isinstance(a, compat_basestring) or isinstance(b, compat_basestring)): - return _js_arith_add(a, b) - if not isinstance(a, compat_basestring): - a = _js_toString(a) - elif not isinstance(b, compat_basestring): - b = _js_toString(b) - return operator.concat(a, b) - - -_js_mod = _js_arith_op(operator.mod) -__js_exp = _js_arith_op(operator.pow) +def _js_mod(a, b): + if JS_Undefined in (a, b) or not b: + return _NaN + return (a or 0) % b def _js_exp(a, b): if not b: return 1 # even 0 ** 0 !! - return __js_exp(a, b) + elif JS_Undefined in (a, b): + return _NaN + return (a or 0) ** b -def _js_to_primitive(v): - return ( - ','.join(map(_js_toString, v)) if isinstance(v, list) - else '[object Object]' if isinstance(v, dict) - else compat_str(v) if not isinstance(v, ( - compat_numeric_types, compat_basestring)) - else v - ) - - -# more exact: yt-dlp/yt-dlp#12110 -def _js_toString(v): - return ( - 'undefined' if v is JS_Undefined - else 'Infinity' if v == _Infinity - else 'NaN' if v is _NaN - else 'null' if v is None - # bool <= int: do this first - else ('false', 'true')[v] if isinstance(v, bool) - else re.sub(r'(?<=\d)\.?0*$', '', '{0:.7f}'.format(v)) if isinstance(v, compat_numeric_types) - else _js_to_primitive(v)) - - -_nullish = frozenset((None, JS_Undefined)) - - -def _js_eq(a, b): - # NaN != any - if _NaN in (a, b): - return False - # Object is Object - if isinstance(a, type(b)) and isinstance(b, (dict, list)): - return operator.is_(a, b) - # general case - if a == b: - return True - # null == undefined - a_b = set((a, b)) - if a_b & _nullish: - return a_b <= _nullish - a, b = _js_to_primitive(a), _js_to_primitive(b) - if not isinstance(a, compat_basestring): - a, b = b, a - # Number to String: convert the string to a number - # Conversion failure results in ... false - if isinstance(a, compat_basestring): - return float_or_none(a) == b - return a == b - - -def _js_neq(a, b): - return not _js_eq(a, b) - - -def _js_id_op(op): +def _js_eq_op(op): @wraps_op(op) def wrapped(a, b): - if _NaN in (a, b): - return op(_NaN, None) - if not isinstance(a, (compat_basestring, compat_numeric_types)): - a, b = b, a - # strings are === if == - # why 'a' is not 'a': https://stackoverflow.com/a/1504848 - if isinstance(a, (compat_basestring, compat_numeric_types)): - return a == b if op(0, 0) else a != b + if set((a, b)) <= set((None, JS_Undefined)): + return op(a, a) return op(a, b) return wrapped @@ -237,52 +138,31 @@ def _js_ternary(cndn, if_true=True, if_false=False): return if_true -def _js_unary_op(op): - - @wraps_op(op) - def wrapped(a, _): - return op(a) - - return wrapped - - -# https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/typeof -def _js_typeof(expr): - with compat_contextlib_suppress(TypeError, KeyError): - return { - JS_Undefined: 'undefined', - _NaN: 'number', - _Infinity: 'number', - True: 'boolean', - False: 'boolean', - None: 'object', - }[expr] - for t, n in ( - (compat_basestring, 'string'), - (compat_numeric_types, 'number'), - ): - if isinstance(expr, t): - return n - if callable(expr): - return 'function' - # TODO: Symbol, BigInt - return 'object' - - # (op, definition) in order of binding priority, tightest first # avoid dict to maintain order # definition None => Defined in JSInterpreter._operator _OPERATORS = ( - ('>>', _js_bit_op(operator.rshift, True)), - ('<<', _js_bit_op(operator.lshift, True)), - ('+', _js_add), + ('>>', _js_bit_op(operator.rshift)), + ('<<', _js_bit_op(operator.lshift)), + ('+', _js_arith_op(operator.add)), ('-', _js_arith_op(operator.sub)), ('*', _js_arith_op(operator.mul)), ('%', _js_mod), - ('/', _js_arith_op(operator.truediv, div=True)), + ('/', _js_div), ('**', _js_exp), ) +_COMP_OPERATORS = ( + ('===', operator.is_), + ('!==', operator.is_not), + ('==', _js_eq_op(operator.eq)), + ('!=', _js_eq_op(operator.ne)), + ('<=', _js_comp_op(operator.le)), + ('>=', _js_comp_op(operator.ge)), + ('<', _js_comp_op(operator.lt)), + ('>', _js_comp_op(operator.gt)), +) + _LOG_OPERATORS = ( ('|', _js_bit_op(operator.or_)), ('^', _js_bit_op(operator.xor)), @@ -296,30 +176,15 @@ _SC_OPERATORS = ( ('&&', None), ) -_UNARY_OPERATORS_X = ( - ('void', _js_unary_op(lambda _: JS_Undefined)), - ('typeof', _js_unary_op(_js_typeof)), - # avoid functools.partial here since Py2 update_wrapper(partial) -> no __module__ - ('!', _js_unary_op(lambda x: _js_ternary(x, if_true=False, if_false=True))), -) - -_COMP_OPERATORS = ( - ('===', _js_id_op(operator.is_)), - ('!==', _js_id_op(operator.is_not)), - ('==', _js_eq), - ('!=', _js_neq), - ('<=', _js_comp_op(operator.le)), - ('>=', _js_comp_op(operator.ge)), - ('<', _js_comp_op(operator.lt)), - ('>', _js_comp_op(operator.gt)), -) - -_OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS + _SC_OPERATORS)) +_OPERATOR_RE = '|'.join(map(lambda x: re.escape(x[0]), _OPERATORS + _LOG_OPERATORS)) _NAME_RE = r'[a-zA-Z_$][\w$]*' _MATCHING_PARENS = dict(zip(*zip('()', '{}', '[]'))) _QUOTES = '\'"/' -_NESTED_BRACKETS = r'[^[\]]+(?:\[[^[\]]+(?:\[[^\]]+\])?\])?' + + +class JS_Undefined(object): + pass class JS_Break(ExtractorError): @@ -356,7 +221,7 @@ class LocalNameSpace(ChainMap): raise NotImplementedError('Deleting is not supported') def __repr__(self): - return 'LocalNameSpace({0!r})'.format(self.maps) + return 'LocalNameSpace%s' % (self.maps, ) class Debugger(object): @@ -377,10 +242,6 @@ class Debugger(object): @classmethod def wrap_interpreter(cls, f): - if not cls.ENABLED: - return f - - @wraps(f) def interpret_statement(self, stmt, local_vars, allow_recursion, *args, **kwargs): if cls.ENABLED and stmt.strip(): cls.write(stmt, level=allow_recursion) @@ -394,7 +255,7 @@ class Debugger(object): raise if cls.ENABLED and stmt.strip(): if should_ret or repr(ret) != stmt: - cls.write(['->', '=>'][bool(should_ret)], repr(ret), '<-|', stmt, level=allow_recursion) + cls.write(['->', '=>'][should_ret], repr(ret), '<-|', stmt, level=allow_recursion) return ret, should_ret return interpret_statement @@ -415,28 +276,14 @@ class JSInterpreter(object): class Exception(ExtractorError): def __init__(self, msg, *args, **kwargs): expr = kwargs.pop('expr', None) - msg = str_or_none(msg, default='"None"') if expr is not None: msg = '{0} in: {1!r:.100}'.format(msg.rstrip(), expr) super(JSInterpreter.Exception, self).__init__(msg, *args, **kwargs) - class JS_Object(object): - def __getitem__(self, key): - if hasattr(self, key): - return getattr(self, key) - raise KeyError(key) - - def dump(self): - """Serialise the instance""" - raise NotImplementedError - - class JS_RegExp(JS_Object): + class JS_RegExp(object): RE_FLAGS = { # special knowledge: Python's re flags are bitmask values, current max 128 # invent new bitmask values well above that for literal parsing - # JS 'u' flag is effectively always set (surrogate pairs aren't seen), - # but \u{...} and \p{...} escapes aren't handled); no additional JS 'v' - # features are supported # TODO: execute matches with these flags (remaining: d, y) 'd': 1024, # Generate indices for substring matches 'g': 2048, # Global search @@ -444,31 +291,21 @@ class JSInterpreter(object): 'm': re.M, # Multi-line search 's': re.S, # Allows . to match newline characters 'u': re.U, # Treat a pattern as a sequence of unicode code points - 'v': re.U, # Like 'u' with extended character class and \p{} syntax 'y': 4096, # Perform a "sticky" search that matches starting at the current position in the target string } def __init__(self, pattern_txt, flags=0): if isinstance(flags, compat_str): flags, _ = self.regex_flags(flags) + # First, avoid https://github.com/python/cpython/issues/74534 self.__self = None - pattern_txt = str_or_none(pattern_txt) or '(?:)' - # escape unintended embedded flags - pattern_txt = re.sub( - r'(\(\?)([aiLmsux]*)(-[imsx]+:|(? 1..12 - ms = args[6] - for i in range(6, 9): - args[i] = -1 # don't know - if is_utc: - args[-1] = 1 - # TODO: [MDN] When a segment overflows or underflows its expected - # range, it usually "carries over to" or "borrows from" the higher segment. - try: - mktime = calendar.timegm if is_utc else time.mktime - return mktime(time.struct_time(args)) * 1000 + ms - except (OverflowError, ValueError): - return None - - @classmethod - def UTC(cls, *args): - t = cls.__ymd_etc(*args, is_utc=True) - return _NaN if t is None else t - - @staticmethod - def parse(date_str, **kw_is_raw): - is_raw = kw_is_raw.get('is_raw', False) - - t = unified_timestamp(str_or_none(date_str), False) - return int(t * 1000) if t is not None else t if is_raw else _NaN - - @staticmethod - def now(**kw_is_raw): - is_raw = kw_is_raw.get('is_raw', False) - - t = time.time() - return int(t * 1000) if t is not None else t if is_raw else _NaN - - def __init__(self, *args): - if not args: - args = [self.now(is_raw=True)] - if len(args) == 1: - if isinstance(args[0], JSInterpreter.JS_Date): - self._t = int_or_none(args[0].valueOf(), default=None) - else: - arg_type = _js_typeof(args[0]) - if arg_type == 'string': - self._t = self.parse(args[0], is_raw=True) - elif arg_type == 'number': - self._t = int(args[0]) - else: - self._t = self.__ymd_etc(*args) - - def toString(self): - try: - return time.strftime('%a %b %0d %Y %H:%M:%S %Z%z', self._t).rstrip() - except TypeError: - return "Invalid Date" - - def valueOf(self): - return _NaN if self._t is None else self._t - - def dump(self): - return '(new Date({0}))'.format(self.toString()) - @classmethod def __op_chars(cls): op_chars = set(';,[') for op in cls._all_operators(): - if op[0].isalpha(): - continue op_chars.update(op[0]) return op_chars @@ -632,18 +369,9 @@ class JSInterpreter(object): skipping = 0 if skip_delims: skip_delims = variadic(skip_delims) - skip_txt = None for idx, char in enumerate(expr): - if skip_txt and idx <= skip_txt[1]: - continue paren_delta = 0 if not in_quote: - if char == '/' and expr[idx:idx + 2] == '/*': - # skip a comment - skip_txt = expr[idx:].find('*/', 2) - skip_txt = [idx, idx + skip_txt + 1] if skip_txt >= 2 else None - if skip_txt: - continue if char in _MATCHING_PARENS: counters[_MATCHING_PARENS[char]] += 1 paren_delta = 1 @@ -676,19 +404,12 @@ class JSInterpreter(object): if pos < delim_len: pos += 1 continue - if skip_txt and skip_txt[0] >= start and skip_txt[1] <= idx - delim_len: - yield expr[start:skip_txt[0]] + expr[skip_txt[1] + 1: idx - delim_len] - else: - yield expr[start: idx - delim_len] - skip_txt = None + yield expr[start: idx - delim_len] start, pos = idx + 1, 0 splits += 1 if max_split and splits >= max_split: break - if skip_txt and skip_txt[0] >= start: - yield expr[start:skip_txt[0]] + expr[skip_txt[1] + 1:] - else: - yield expr[start:] + yield expr[start:] @classmethod def _separate_at_paren(cls, expr, delim=None): @@ -704,71 +425,9 @@ class JSInterpreter(object): if not _cached: _cached.extend(itertools.chain( # Ref: https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Operators/Operator_Precedence - _SC_OPERATORS, _LOG_OPERATORS, _COMP_OPERATORS, _OPERATORS, _UNARY_OPERATORS_X)) + _SC_OPERATORS, _LOG_OPERATORS, _COMP_OPERATORS, _OPERATORS)) return _cached - def _separate_at_op(self, expr, max_split=None): - - for op, _ in self._all_operators(): - # hackety: have higher priority than <>, but don't confuse them - skip_delim = (op + op) if op in '<>*?' else None - if op == '?': - skip_delim = (skip_delim, '?.') - separated = list(self._separate(expr, op, skip_delims=skip_delim)) - if len(separated) < 2: - continue - - right_expr = separated.pop() - # handle operators that are both unary and binary, minimal BODMAS - if op in ('+', '-'): - # simplify/adjust consecutive instances of these operators - undone = 0 - separated = [s.strip() for s in separated] - while len(separated) > 1 and not separated[-1]: - undone += 1 - separated.pop() - if op == '-' and undone % 2 != 0: - right_expr = op + right_expr - elif op == '+': - while len(separated) > 1 and set(separated[-1]) <= self.OP_CHARS: - right_expr = separated.pop() + right_expr - if separated[-1][-1:] in self.OP_CHARS: - right_expr = separated.pop() + right_expr - # hanging op at end of left => unary + (strip) or - (push right) - separated.append(right_expr) - dm_ops = ('*', '%', '/', '**') - dm_chars = set(''.join(dm_ops)) - - def yield_terms(s): - skip = False - for i, term in enumerate(s[:-1]): - if skip: - skip = False - continue - if not (dm_chars & set(term)): - yield term - continue - for dm_op in dm_ops: - bodmas = list(self._separate(term, dm_op, skip_delims=skip_delim)) - if len(bodmas) > 1 and not bodmas[-1].strip(): - bodmas[-1] = (op if op == '-' else '') + s[i + 1] - yield dm_op.join(bodmas) - skip = True - break - else: - if term: - yield term - - if not skip and s[-1]: - yield s[-1] - - separated = list(yield_terms(separated)) - right_expr = separated.pop() if len(separated) > 1 else None - expr = op.join(separated) - if right_expr is None: - continue - return op, separated, right_expr - def _operator(self, op, left_val, right_expr, expr, local_vars, allow_recursion): if op in ('||', '&&'): if (op == '&&') ^ _js_ternary(left_val): @@ -779,7 +438,7 @@ class JSInterpreter(object): elif op == '?': right_expr = _js_ternary(left_val, *self._separate(right_expr, ':', 1)) - right_val = self.interpret_expression(right_expr, local_vars, allow_recursion) if right_expr else left_val + right_val = self.interpret_expression(right_expr, local_vars, allow_recursion) opfunc = op and next((v for k, v in self._all_operators() if k == op), None) if not opfunc: return right_val @@ -790,21 +449,17 @@ class JSInterpreter(object): except Exception as e: raise self.Exception('Failed to evaluate {left_val!r:.50} {op} {right_val!r:.50}'.format(**locals()), expr, cause=e) - def _index(self, obj, idx, allow_undefined=None): - if idx == 'length' and isinstance(obj, list): + def _index(self, obj, idx, allow_undefined=False): + if idx == 'length': return len(obj) try: - return obj[int(idx)] if isinstance(obj, list) else obj[compat_str(idx)] - except (TypeError, KeyError, IndexError, ValueError) as e: - # allow_undefined is None gives correct behaviour - if allow_undefined or ( - allow_undefined is None and not isinstance(e, TypeError)): + return obj[int(idx)] if isinstance(obj, list) else obj[idx] + except Exception as e: + if allow_undefined: return JS_Undefined raise self.Exception('Cannot get index {idx!r:.100}'.format(**locals()), expr=repr(obj), cause=e) def _dump(self, obj, namespace): - if obj is JS_Undefined: - return 'undefined' try: return json.dumps(obj) except TypeError: @@ -812,7 +467,7 @@ class JSInterpreter(object): # used below _VAR_RET_THROW_RE = re.compile(r'''(?x) - (?:(?Pvar|const|let)\s+|(?Preturn)(?:\s+|(?=["'])|$)|(?Pthrow)\s+) + (?P(?:var|const|let)\s)|return(?:\s+|(?=["'])|$)|(?Pthrow\s+) ''') _COMPOUND_RE = re.compile(r'''(?x) (?Ptry)\s*\{| @@ -824,10 +479,6 @@ class JSInterpreter(object): _FINALLY_RE = re.compile(r'finally\s*\{') _SWITCH_RE = re.compile(r'switch\s*\(') - def _eval_operator(self, op, left_expr, right_expr, expr, local_vars, allow_recursion): - left_val = self.interpret_expression(left_expr, local_vars, allow_recursion) - return self._operator(op, left_val, right_expr, expr, local_vars, allow_recursion) - @Debugger.wrap_interpreter def interpret_statement(self, stmt, local_vars, allow_recursion=100): if allow_recursion < 0: @@ -850,7 +501,7 @@ class JSInterpreter(object): expr = stmt[len(m.group(0)):].strip() if m.group('throw'): raise JS_Throw(self.interpret_expression(expr, local_vars, allow_recursion)) - should_return = 'return' if m.group('ret') else False + should_return = not m.group('var') if not expr: return None, should_return @@ -867,7 +518,7 @@ class JSInterpreter(object): new_kw, _, obj = expr.partition('new ') if not new_kw: - for klass, konstr in (('Date', lambda *x: self.JS_Date(*x).valueOf()), + for klass, konstr in (('Date', lambda x: int(unified_timestamp(x, False) * 1000)), ('RegExp', self.JS_RegExp), ('Error', self.Exception)): if not obj.startswith(klass + '('): @@ -882,19 +533,9 @@ class JSInterpreter(object): else: raise self.Exception('Unsupported object {obj:.100}'.format(**locals()), expr=expr) - # apply unary operators (see new above) - for op, _ in _UNARY_OPERATORS_X: - if not expr.startswith(op): - continue - operand = expr[len(op):] - if not operand or (op.isalpha() and operand[0] != ' '): - continue - separated = self._separate_at_op(operand, max_split=1) - if separated: - next_op, separated, right_expr = separated - separated.append(right_expr) - operand = next_op.join(separated) - return self._eval_operator(op, operand, '', expr, local_vars, allow_recursion), should_return + if expr.startswith('void '): + left = self.interpret_expression(expr[5:], local_vars, allow_recursion) + return None, should_return if expr.startswith('{'): inner, outer = self._separate_at_paren(expr) @@ -941,7 +582,7 @@ class JSInterpreter(object): if_expr, expr = self._separate_at_paren(expr) else: # may lose ... else ... because of ll.368-374 - if_expr, expr = self._separate_at_paren(' %s;' % (expr,), delim=';') + if_expr, expr = self._separate_at_paren(expr, delim=';') else_expr = None m = re.match(r'else\s*(?P\{)?', expr) if m: @@ -1079,7 +720,7 @@ class JSInterpreter(object): start, end = m.span() sign = m.group('pre_sign') or m.group('post_sign') ret = local_vars[var] - local_vars[var] = _js_add(ret, 1 if sign[0] == '+' else -1) + local_vars[var] += 1 if sign[0] == '+' else -1 if m.group('pre_sign'): ret = local_vars[var] expr = expr[:start] + self._dump(ret, local_vars) + expr[end:] @@ -1089,18 +730,15 @@ class JSInterpreter(object): m = re.match(r'''(?x) (?P - (?P{_NAME_RE})(?P(?:\[{_NESTED_BRACKETS}\])+)?\s* + (?P{_NAME_RE})(?:\[(?P[^\]]+?)\])?\s* (?P{_OPERATOR_RE})? =(?!=)(?P.*)$ )|(?P (?!if|return|true|false|null|undefined|NaN|Infinity)(?P{_NAME_RE})$ - )|(?P - (?P{_NAME_RE})(?: - (?P\?)?\.(?P[^(]+)| - \[(?P{_NESTED_BRACKETS})\] - )\s* )|(?P - (?P{_NAME_RE})(?P\[.+\])$ + (?P{_NAME_RE})\[(?P.+)\]$ + )|(?P + (?P{_NAME_RE})(?:(?P\?)?\.(?P[^(]+)|\[(?P[^\]]+)\])\s* )|(?P (?P{_NAME_RE})\((?P.*)\)$ )'''.format(**globals()), expr) @@ -1108,28 +746,19 @@ class JSInterpreter(object): if md.get('assign'): left_val = local_vars.get(m.group('out')) - if not m.group('out_idx'): + if not m.group('index'): local_vars[m.group('out')] = self._operator( m.group('op'), left_val, m.group('expr'), expr, local_vars, allow_recursion) return local_vars[m.group('out')], should_return elif left_val in (None, JS_Undefined): raise self.Exception('Cannot index undefined variable ' + m.group('out'), expr=expr) - indexes = md['out_idx'] - while indexes: - idx, indexes = self._separate_at_paren(indexes) - idx = self.interpret_expression(idx, local_vars, allow_recursion) - if indexes: - left_val = self._index(left_val, idx) - if isinstance(idx, float): - idx = int(idx) - if isinstance(left_val, list) and len(left_val) <= int_or_none(idx, default=-1): - # JS Array is a sparsely assignable list - # TODO: handle extreme sparsity without memory bloat, eg using auxiliary dict - left_val.extend((idx - len(left_val) + 1) * [JS_Undefined]) + idx = self.interpret_expression(m.group('index'), local_vars, allow_recursion) + if not isinstance(idx, (int, float)): + raise self.Exception('List index %s must be integer' % (idx, ), expr=expr) + idx = int(idx) left_val[idx] = self._operator( - m.group('op'), self._index(left_val, idx) if m.group('op') else None, - m.group('expr'), expr, local_vars, allow_recursion) + m.group('op'), self._index(left_val, idx), m.group('expr'), expr, local_vars, allow_recursion) return left_val[idx], should_return elif expr.isdigit(): @@ -1147,34 +776,63 @@ class JSInterpreter(object): return _Infinity, should_return elif md.get('return'): - ret = local_vars[m.group('name')] - # challenge may try to force returning the original value - # use an optional internal var to block this - if should_return == 'return': - if '_ytdl_do_not_return' not in local_vars: - return ret, True - return (ret, True) if ret != local_vars['_ytdl_do_not_return'] else (ret, False) - else: - return ret, should_return + return local_vars[m.group('name')], should_return - with compat_contextlib_suppress(ValueError): + try: ret = json.loads(js_to_json(expr)) # strict=True) if not md.get('attribute'): return ret, should_return + except ValueError: + pass if md.get('indexing'): val = local_vars[m.group('in')] - indexes = m.group('in_idx') - while indexes: - idx, indexes = self._separate_at_paren(indexes) - idx = self.interpret_expression(idx, local_vars, allow_recursion) - val = self._index(val, idx) - return val, should_return + idx = self.interpret_expression(m.group('idx'), local_vars, allow_recursion) + return self._index(val, idx), should_return - separated = self._separate_at_op(expr) - if separated: - op, separated, right_expr = separated - return self._eval_operator(op, op.join(separated), right_expr, expr, local_vars, allow_recursion), should_return + for op, _ in self._all_operators(): + # hackety: have higher priority than <>, but don't confuse them + skip_delim = (op + op) if op in '<>*?' else None + if op == '?': + skip_delim = (skip_delim, '?.') + separated = list(self._separate(expr, op, skip_delims=skip_delim)) + if len(separated) < 2: + continue + + right_expr = separated.pop() + # handle operators that are both unary and binary, minimal BODMAS + if op in ('+', '-'): + # simplify/adjust consecutive instances of these operators + undone = 0 + separated = [s.strip() for s in separated] + while len(separated) > 1 and not separated[-1]: + undone += 1 + separated.pop() + if op == '-' and undone % 2 != 0: + right_expr = op + right_expr + elif op == '+': + while len(separated) > 1 and set(separated[-1]) <= self.OP_CHARS: + right_expr = separated.pop() + right_expr + if separated[-1][-1:] in self.OP_CHARS: + right_expr = separated.pop() + right_expr + # hanging op at end of left => unary + (strip) or - (push right) + left_val = separated[-1] if separated else '' + for dm_op in ('*', '%', '/', '**'): + bodmas = tuple(self._separate(left_val, dm_op, skip_delims=skip_delim)) + if len(bodmas) > 1 and not bodmas[-1].strip(): + expr = op.join(separated) + op + right_expr + if len(separated) > 1: + separated.pop() + right_expr = op.join((left_val, right_expr)) + else: + separated = [op.join((left_val, right_expr))] + right_expr = None + break + if right_expr is None: + continue + + left_val = self.interpret_expression(op.join(separated), local_vars, allow_recursion) + return self._operator(op, left_val, right_expr, expr, local_vars, allow_recursion), should_return if md.get('attribute'): variable, member, nullish = m.group('var', 'member', 'nullish') @@ -1195,15 +853,12 @@ class JSInterpreter(object): def eval_method(variable, member): if (variable, member) == ('console', 'debug'): if Debugger.ENABLED: - Debugger.write(self.interpret_expression('[{0}]'.format(arg_str), local_vars, allow_recursion)) + Debugger.write(self.interpret_expression('[{}]'.format(arg_str), local_vars, allow_recursion)) return types = { 'String': compat_str, 'Math': float, 'Array': list, - 'Date': self.JS_Date, - 'RegExp': self.JS_RegExp, - # 'Error': self.Exception, # has no std static methods } obj = local_vars.get(variable) if obj in (JS_Undefined, None): @@ -1211,7 +866,7 @@ class JSInterpreter(object): if obj is JS_Undefined: try: if variable not in self._objects: - self._objects[variable] = self.extract_object(variable, local_vars) + self._objects[variable] = self.extract_object(variable) obj = self._objects[variable] except self.Exception: if not nullish: @@ -1249,105 +904,67 @@ class JSInterpreter(object): if obj is compat_str: if member == 'fromCharCode': assertion(argvals, 'takes one or more arguments') - return ''.join(compat_chr(int(n)) for n in argvals) + return ''.join(map(compat_chr, argvals)) raise self.Exception('Unsupported string method ' + member, expr=expr) elif obj is float: if member == 'pow': assertion(len(argvals) == 2, 'takes two arguments') return argvals[0] ** argvals[1] raise self.Exception('Unsupported Math method ' + member, expr=expr) - elif obj is self.JS_Date: - return getattr(obj, member)(*argvals) if member == 'split': - assertion(len(argvals) <= 2, 'takes at most two arguments') - if len(argvals) > 1: - limit = argvals[1] - assertion(isinstance(limit, int) and limit >= 0, 'integer limit >= 0') - if limit == 0: - return [] - else: - limit = 0 - if len(argvals) == 0: - argvals = [JS_Undefined] - elif isinstance(argvals[0], self.JS_RegExp): - # avoid re.split(), similar but not enough - - def where(): - for m in argvals[0].finditer(obj): - yield m.span(0) - yield (None, None) - - def splits(limit=limit): - i = 0 - for j, jj in where(): - if j == jj == 0: - continue - if j is None and i >= len(obj): - break - yield obj[i:j] - if jj is None or limit == 1: - break - limit -= 1 - i = jj - - return list(splits()) - return ( - obj.split(argvals[0], limit - 1) if argvals[0] and argvals[0] != JS_Undefined - else list(obj)[:limit or None]) + assertion(argvals, 'takes one or more arguments') + assertion(len(argvals) == 1, 'with limit argument is not implemented') + return obj.split(argvals[0]) if argvals[0] else list(obj) elif member == 'join': assertion(isinstance(obj, list), 'must be applied on a list') - assertion(len(argvals) <= 1, 'takes at most one argument') - return (',' if len(argvals) == 0 or argvals[0] in (None, JS_Undefined) - else argvals[0]).join( - ('' if x in (None, JS_Undefined) else _js_toString(x)) - for x in obj) + assertion(len(argvals) == 1, 'takes exactly one argument') + return argvals[0].join(obj) elif member == 'reverse': assertion(not argvals, 'does not take any arguments') obj.reverse() return obj elif member == 'slice': - assertion(isinstance(obj, (list, compat_str)), 'must be applied on a list or string') - # From [1]: - # .slice() - like [:] - # .slice(n) - like [n:] (not [slice(n)] - # .slice(m, n) - like [m:n] or [slice(m, n)] - # [1] https://developer.mozilla.org/en-US/docs/Web/JavaScript/Reference/Global_Objects/Array/slice - assertion(len(argvals) <= 2, 'takes between 0 and 2 arguments') - if len(argvals) < 2: - argvals += (None,) - return obj[slice(*argvals)] + assertion(isinstance(obj, list), 'must be applied on a list') + assertion(len(argvals) == 1, 'takes exactly one argument') + return obj[argvals[0]:] elif member == 'splice': assertion(isinstance(obj, list), 'must be applied on a list') assertion(argvals, 'takes one or more arguments') index, how_many = map(int, (argvals + [len(obj)])[:2]) if index < 0: index += len(obj) - res = [obj.pop(index) - for _ in range(index, min(index + how_many, len(obj)))] - obj[index:index] = argvals[2:] + add_items = argvals[2:] + res = [] + for _ in range(index, min(index + how_many, len(obj))): + res.append(obj.pop(index)) + for i, item in enumerate(add_items): + obj.insert(index + i, item) return res - elif member in ('shift', 'pop'): - assertion(isinstance(obj, list), 'must be applied on a list') - assertion(not argvals, 'does not take any arguments') - return obj.pop(0 if member == 'shift' else -1) if len(obj) > 0 else JS_Undefined elif member == 'unshift': assertion(isinstance(obj, list), 'must be applied on a list') - # not enforced: assertion(argvals, 'takes one or more arguments') - obj[0:0] = argvals - return len(obj) + assertion(argvals, 'takes one or more arguments') + for item in reversed(argvals): + obj.insert(0, item) + return obj + elif member == 'pop': + assertion(isinstance(obj, list), 'must be applied on a list') + assertion(not argvals, 'does not take any arguments') + if not obj: + return + return obj.pop() elif member == 'push': - # not enforced: assertion(argvals, 'takes one or more arguments') + assertion(argvals, 'takes one or more arguments') obj.extend(argvals) - return len(obj) + return obj elif member == 'forEach': assertion(argvals, 'takes one or more arguments') - assertion(len(argvals) <= 2, 'takes at most 2 arguments') + assertion(len(argvals) <= 2, 'takes at-most 2 arguments') f, this = (argvals + [''])[:2] return [f((item, idx, obj), {'this': this}, allow_recursion) for idx, item in enumerate(obj)] elif member == 'indexOf': assertion(argvals, 'takes one or more arguments') - assertion(len(argvals) <= 2, 'takes at most 2 arguments') + assertion(len(argvals) <= 2, 'takes at-most 2 arguments') idx, start = (argvals + [0])[:2] try: return obj.index(idx, start) @@ -1356,7 +973,7 @@ class JSInterpreter(object): elif member == 'charCodeAt': assertion(isinstance(obj, compat_str), 'must be applied on a string') # assertion(len(argvals) == 1, 'takes exactly one argument') # but not enforced - idx = argvals[0] if len(argvals) > 0 and isinstance(argvals[0], int) else 0 + idx = argvals[0] if isinstance(argvals[0], int) else 0 if idx >= len(obj): return None return ord(obj[idx]) @@ -1365,8 +982,7 @@ class JSInterpreter(object): assertion(len(argvals) == 2, 'takes exactly two arguments') # TODO: argvals[1] callable, other Py vs JS edge cases if isinstance(argvals[0], self.JS_RegExp): - # access JS member with Py reserved name - count = 0 if self._index(argvals[0], 'global') else 1 + count = 0 if argvals[0].flags & self.JS_RegExp.RE_FLAGS['g'] else 1 assertion(member != 'replaceAll' or count == 0, 'replaceAll must be called with a global RegExp') return argvals[0].sub(argvals[1], obj, count=count) @@ -1407,8 +1023,8 @@ class JSInterpreter(object): for v in self._separate(list_txt): yield self.interpret_expression(v, local_vars, allow_recursion) - def extract_object(self, objname, *global_stack): - _FUNC_NAME_RE = r'''(?:{n}|"{n}"|'{n}')'''.format(n=_NAME_RE) + def extract_object(self, objname): + _FUNC_NAME_RE = r'''(?:[a-zA-Z$0-9]+|"[a-zA-Z$0-9]+"|'[a-zA-Z$0-9]+')''' obj = {} fields = next(filter(None, ( obj_m.group('fields') for obj_m in re.finditer( @@ -1428,8 +1044,7 @@ class JSInterpreter(object): fields): argnames = self.build_arglist(f.group('args')) name = remove_quotes(f.group('key')) - obj[name] = function_with_repr( - self.build_function(argnames, f.group('code'), *global_stack), 'F<{0}>'.format(name)) + obj[name] = function_with_repr(self.build_function(argnames, f.group('code')), 'F<{0}>'.format(name)) return obj @@ -1461,31 +1076,27 @@ class JSInterpreter(object): code, _ = self._separate_at_paren(func_m.group('code')) # refine the match return self.build_arglist(func_m.group('args')), code - def extract_function(self, funcname, *global_stack): + def extract_function(self, funcname): return function_with_repr( - self.extract_function_from_code(*itertools.chain( - self.extract_function_code(funcname), global_stack)), + self.extract_function_from_code(*self.extract_function_code(funcname)), 'F<%s>' % (funcname,)) def extract_function_from_code(self, argnames, code, *global_stack): local_vars = {} - - start = None while True: - mobj = re.search(r'function\((?P[^)]*)\)\s*{', code[start:]) + mobj = re.search(r'function\((?P[^)]*)\)\s*{', code) if mobj is None: break - start, body_start = ((start or 0) + x for x in mobj.span()) + start, body_start = mobj.span() body, remaining = self._separate_at_paren(code[body_start - 1:]) name = self._named_object(local_vars, self.extract_function_from_code( [x.strip() for x in mobj.group('args').split(',')], body, local_vars, *global_stack)) code = code[:start] + name + remaining - return self.build_function(argnames, code, local_vars, *global_stack) - def call_function(self, funcname, *args, **kw_global_vars): - return self.extract_function(funcname)(args, kw_global_vars) + def call_function(self, funcname, *args): + return self.extract_function(funcname)(args) @classmethod def build_arglist(cls, arg_text): @@ -1504,9 +1115,8 @@ class JSInterpreter(object): global_stack = list(global_stack) or [{}] argnames = tuple(argnames) - def resf(args, kwargs=None, allow_recursion=100): - kwargs = kwargs or {} - global_stack[0].update(zip_longest(argnames, args, fillvalue=JS_Undefined)) + def resf(args, kwargs={}, allow_recursion=100): + global_stack[0].update(zip_longest(argnames, args, fillvalue=None)) global_stack[0].update(kwargs) var_stack = LocalNameSpace(*global_stack) ret, should_abort = self.interpret_statement(code.replace('\n', ' '), var_stack, allow_recursion - 1) diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index c4262936e..ac1e78002 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -4204,16 +4204,12 @@ def lowercase_escape(s): s) -def escape_rfc3986(s, safe=None): +def escape_rfc3986(s): """Escape non-ASCII characters as suggested by RFC 3986""" if sys.version_info < (3, 0): s = _encode_compat_str(s, 'utf-8') - if safe is not None: - safe = _encode_compat_str(safe, 'utf-8') - if safe is None: - safe = b"%/;:@&=+$,!~*'()?#[]" # ensure unicode: after quoting, it can always be converted - return compat_str(compat_urllib_parse.quote(s, safe)) + return compat_str(compat_urllib_parse.quote(s, b"%/;:@&=+$,!~*'()?#[]")) def escape_url(url): diff --git a/youtube_dl/version.py b/youtube_dl/version.py index c70d9d2af..b82fbc702 100644 --- a/youtube_dl/version.py +++ b/youtube_dl/version.py @@ -1,3 +1,3 @@ from __future__ import unicode_literals -__version__ = '2025.04.07' +__version__ = '2021.12.17'