diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md index e5405c235..febbd2344 100644 --- a/.github/ISSUE_TEMPLATE/1_broken_site.md +++ b/.github/ISSUE_TEMPLATE/1_broken_site.md @@ -18,7 +18,7 @@ title: '' - [ ] I'm reporting a broken site support -- [ ] I've verified that I'm running youtube-dl version **2021.12.17** +- [ ] I've verified that I'm running youtube-dl version **2021.04.07** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar issues including closed ones @@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v < [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version 2021.12.17 + [debug] youtube-dl version 2021.04.07 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md index 33b01ce7f..d7296d0a9 100644 --- a/.github/ISSUE_TEMPLATE/2_site_support_request.md +++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md @@ -19,7 +19,7 @@ labels: 'site-support-request' - [ ] I'm reporting a new site support request -- [ ] I've verified that I'm running youtube-dl version **2021.12.17** +- [ ] I've verified that I'm running youtube-dl version **2021.04.07** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that none of provided URLs violate any copyrights - [ ] I've searched the bugtracker for similar site support requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md index 285610cc7..92e616a1a 100644 --- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md +++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md @@ -18,13 +18,13 @@ title: '' - [ ] I'm reporting a site feature request -- [ ] I've verified that I'm running youtube-dl version **2021.12.17** +- [ ] I've verified that I'm running youtube-dl version **2021.04.07** - [ ] I've searched the bugtracker for similar site feature requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md index af73525fb..b55739f6c 100644 --- a/.github/ISSUE_TEMPLATE/4_bug_report.md +++ b/.github/ISSUE_TEMPLATE/4_bug_report.md @@ -18,7 +18,7 @@ title: '' - [ ] I'm reporting a broken site support issue -- [ ] I've verified that I'm running youtube-dl version **2021.12.17** +- [ ] I've verified that I'm running youtube-dl version **2021.04.07** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar bug reports including closed ones @@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v < [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version 2021.12.17 + [debug] youtube-dl version 2021.04.07 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md index 42c878b83..dbdb8356a 100644 --- a/.github/ISSUE_TEMPLATE/5_feature_request.md +++ b/.github/ISSUE_TEMPLATE/5_feature_request.md @@ -19,13 +19,13 @@ labels: 'request' - [ ] I'm reporting a feature request -- [ ] I've verified that I'm running youtube-dl version **2021.12.17** +- [ ] I've verified that I'm running youtube-dl version **2021.04.07** - [ ] I've searched the bugtracker for similar feature requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml deleted file mode 100644 index 3ba13e0ce..000000000 --- a/.github/ISSUE_TEMPLATE/config.yml +++ /dev/null @@ -1 +0,0 @@ -blank_issues_enabled: false diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8234e0ccb..a9dc47a71 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,482 +1,74 @@ name: CI - -env: - all-cpython-versions: 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 - main-cpython-versions: 2.7, 3.2, 3.5, 3.9, 3.11 - pypy-versions: pypy-2.7, pypy-3.6, pypy-3.7 - cpython-versions: main - test-set: core - # Python beta version to be built using pyenv before setup-python support - # Must also be included in all-cpython-versions - next: 3.13 - -on: - push: - # push inputs aren't known to GitHub - inputs: - cpython-versions: - type: string - default: all - test-set: - type: string - default: core - pull_request: - # pull_request inputs aren't known to GitHub - inputs: - cpython-versions: - type: string - default: main - test-set: - type: string - default: both - workflow_dispatch: - inputs: - cpython-versions: - type: choice - description: CPython versions (main = 2.7, 3.2, 3.5, 3.9, 3.11) - options: - - all - - main - required: true - default: main - test-set: - type: choice - description: core, download - options: - - both - - core - - download - required: true - default: both - -permissions: - contents: read - +on: [push, pull_request] jobs: - select: - name: Select tests from inputs - runs-on: ubuntu-latest - outputs: - cpython-versions: ${{ steps.run.outputs.cpython-versions }} - test-set: ${{ steps.run.outputs.test-set }} - own-pip-versions: ${{ steps.run.outputs.own-pip-versions }} - steps: - # push and pull_request inputs aren't known to GitHub (pt3) - - name: Set push defaults - if: ${{ github.event_name == 'push' }} - env: - cpython-versions: all - test-set: core - run: | - echo "cpython-versions=${{env.cpython-versions}}" >> "$GITHUB_ENV" - echo "test_set=${{env.test_set}}" >> "$GITHUB_ENV" - - name: Get pull_request inputs - if: ${{ github.event_name == 'pull_request' }} - env: - cpython-versions: main - test-set: both - run: | - echo "cpython-versions=${{env.cpython-versions}}" >> "$GITHUB_ENV" - echo "test_set=${{env.test_set}}" >> "$GITHUB_ENV" - - name: Make version array - id: run - run: | - # Make a JSON Array from comma/space-separated string (no extra escaping) - json_list() { \ - ret=""; IFS="${IFS},"; set -- $*; \ - for a in "$@"; do \ - ret=$(printf '%s"%s"' "${ret}${ret:+, }" "$a"); \ - done; \ - printf '[%s]' "$ret"; } - tests="${{ inputs.test-set || env.test-set }}" - [ $tests = both ] && tests="core download" - printf 'test-set=%s\n' "$(json_list $tests)" >> "$GITHUB_OUTPUT" - versions="${{ inputs.cpython-versions || env.cpython-versions }}" - if [ "$versions" = all ]; then \ - versions="${{ env.all-cpython-versions }}"; else \ - versions="${{ env.main-cpython-versions }}"; \ - fi - printf 'cpython-versions=%s\n' \ - "$(json_list ${versions}${versions:+, }${{ env.pypy-versions }})" >> "$GITHUB_OUTPUT" - # versions with a special get-pip.py in a per-version subdirectory - printf 'own-pip-versions=%s\n' \ - "$(json_list 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6)" >> "$GITHUB_OUTPUT" - tests: - name: Run tests - needs: select - permissions: - contents: read - packages: write + name: Tests runs-on: ${{ matrix.os }} - env: - PIP: python -m pip - PIP_DISABLE_PIP_VERSION_CHECK: true - PIP_NO_PYTHON_VERSION_WARNING: true strategy: fail-fast: true matrix: - os: [ubuntu-22.04] - python-version: ${{ fromJSON(needs.select.outputs.cpython-versions) }} + os: [ubuntu-18.04] + # TODO: python 2.6 + python-version: [2.7, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, pypy-2.7, pypy-3.6, pypy-3.7] python-impl: [cpython] - ytdl-test-set: ${{ fromJSON(needs.select.outputs.test-set) }} + ytdl-test-set: [core, download] run-tests-ext: [sh] include: - - os: windows-2019 - python-version: 3.4 + # python 3.2 is only available on windows via setup-python + - os: windows-latest + python-version: 3.2 python-impl: cpython - ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }} + ytdl-test-set: core run-tests-ext: bat - - os: windows-2019 - python-version: 3.4 + - os: windows-latest + python-version: 3.2 python-impl: cpython - ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download' || 'nodownload' }} + ytdl-test-set: download run-tests-ext: bat # jython - - os: ubuntu-22.04 - python-version: 2.7 + - os: ubuntu-18.04 python-impl: jython - ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }} + ytdl-test-set: core run-tests-ext: sh - - os: ubuntu-22.04 - python-version: 2.7 + - os: ubuntu-18.04 python-impl: jython - ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download' || 'nodownload' }} + ytdl-test-set: download run-tests-ext: sh steps: - - name: Prepare Linux - if: ${{ startswith(matrix.os, 'ubuntu') }} - shell: bash - run: | - # apt in runner, if needed, may not be up-to-date - sudo apt-get update - - name: Checkout - uses: actions/checkout@v3 - #-------- Python 3 ----- - - name: Set up supported Python ${{ matrix.python-version }} - id: setup-python - if: ${{ matrix.python-impl == 'cpython' && matrix.python-version != '2.6' && matrix.python-version != '2.7' && matrix.python-version != env.next }} - # wrap broken actions/setup-python@v4 - # NB may run apt-get install in Linux - uses: ytdl-org/setup-python@v1 - env: - # Temporary (?) workaround for Python 3.5 failures - May 2024 - PIP_TRUSTED_HOST: "pypi.python.org pypi.org files.pythonhosted.org" + - uses: actions/checkout@v2 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v2 + if: ${{ matrix.python-impl == 'cpython' }} with: python-version: ${{ matrix.python-version }} - cache-build: true - allow-build: info - - name: Locate supported Python ${{ matrix.python-version }} - if: ${{ env.pythonLocation }} - shell: bash - run: | - echo "PYTHONHOME=${pythonLocation}" >> "$GITHUB_ENV" - export expected="${{ steps.setup-python.outputs.python-path }}" - dirname() { printf '%s\n' \ - 'import os, sys' \ - 'print(os.path.dirname(sys.argv[1]))' \ - | ${expected} - "$1"; } - expd="$(dirname "$expected")" - export python="$(command -v python)" - [ "$expd" = "$(dirname "$python")" ] || echo "PATH=$expd:${PATH}" >> "$GITHUB_ENV" - [ -x "$python" ] || printf '%s\n' \ - 'import os' \ - 'exp = os.environ["expected"]' \ - 'python = os.environ["python"]' \ - 'exps = os.path.split(exp)' \ - 'if python and (os.path.dirname(python) == exp[0]):' \ - ' exit(0)' \ - 'exps[1] = "python" + os.path.splitext(exps[1])[1]' \ - 'python = os.path.join(*exps)' \ - 'try:' \ - ' os.symlink(exp, python)' \ - 'except AttributeError:' \ - ' os.rename(exp, python)' \ - | ${expected} - - printf '%s\n' \ - 'import sys' \ - 'print(sys.path)' \ - | ${expected} - - #-------- Python next (was 3.12) - - - name: Set up CPython 3.next environment - if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next }} - shell: bash - run: | - PYENV_ROOT=$HOME/.local/share/pyenv - echo "PYENV_ROOT=${PYENV_ROOT}" >> "$GITHUB_ENV" - - name: Cache Python 3.next - id: cachenext - if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next }} - uses: actions/cache@v3 - with: - key: python-${{ env.next }} - path: | - ${{ env.PYENV_ROOT }} - - name: Build and set up Python 3.next - if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next && ! steps.cachenext.outputs.cache-hit }} - # dl and build locally - shell: bash - run: | - # Install build environment - sudo apt-get install -y build-essential llvm libssl-dev tk-dev \ - libncursesw5-dev libreadline-dev libsqlite3-dev \ - libffi-dev xz-utils zlib1g-dev libbz2-dev liblzma-dev - # Download PyEnv from its GitHub repository. - export PYENV_ROOT=${{ env.PYENV_ROOT }} - export PATH=$PYENV_ROOT/bin:$PATH - git clone "https://github.com/pyenv/pyenv.git" "$PYENV_ROOT" - pyenv install ${{ env.next }} - - name: Locate Python 3.next - if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next }} - shell: bash - run: | - PYTHONHOME="$(echo "${{ env.PYENV_ROOT }}/versions/${{ env.next }}."*)" - test -n "$PYTHONHOME" - echo "PYTHONHOME=$PYTHONHOME" >> "$GITHUB_ENV" - echo "PATH=${PYTHONHOME}/bin:$PATH" >> "$GITHUB_ENV" - #-------- Python 2.7 -- - - name: Set up Python 2.7 - if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.7' }} - # install 2.7 - shell: bash - run: | - # Ubuntu 22.04 no longer has python-is-python2: fetch it - curl -L "http://launchpadlibrarian.net/474693132/python-is-python2_2.7.17-4_all.deb" -o python-is-python2.deb - sudo apt-get install -y python2 - sudo dpkg --force-breaks -i python-is-python2.deb - echo "PYTHONHOME=/usr" >> "$GITHUB_ENV" - #-------- Python 2.6 -- - - name: Set up Python 2.6 environment - if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' }} - shell: bash - run: | - openssl_name=openssl-1.0.2u - echo "openssl_name=${openssl_name}" >> "$GITHUB_ENV" - openssl_dir=$HOME/.local/opt/$openssl_name - echo "openssl_dir=${openssl_dir}" >> "$GITHUB_ENV" - PYENV_ROOT=$HOME/.local/share/pyenv - echo "PYENV_ROOT=${PYENV_ROOT}" >> "$GITHUB_ENV" - sudo apt-get install -y openssl ca-certificates - - name: Cache Python 2.6 - id: cache26 - if: ${{ matrix.python-version == '2.6' }} - uses: actions/cache@v3 - with: - key: python-2.6.9 - path: | - ${{ env.openssl_dir }} - ${{ env.PYENV_ROOT }} - - name: Build and set up Python 2.6 - if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' && ! steps.cache26.outputs.cache-hit }} - # dl and build locally - shell: bash - run: | - # Install build environment - sudo apt-get install -y build-essential llvm libssl-dev tk-dev \ - libncursesw5-dev libreadline-dev libsqlite3-dev \ - libffi-dev xz-utils zlib1g-dev libbz2-dev liblzma-dev - # Download and install OpenSSL 1.0.2, back in time - openssl_name=${{ env.openssl_name }} - openssl_targz=${openssl_name}.tar.gz - openssl_dir=${{ env.openssl_dir }} - openssl_inc=$openssl_dir/include - openssl_lib=$openssl_dir/lib - openssl_ssl=$openssl_dir/ssl - curl -L "https://www.openssl.org/source/$openssl_targz" -o $openssl_targz - tar -xf $openssl_targz - ( cd $openssl_name; \ - ./config --prefix=$openssl_dir --openssldir=${openssl_dir}/ssl \ - --libdir=lib -Wl,-rpath=${openssl_dir}/lib shared zlib-dynamic && \ - make && \ - make install ) - rm -rf $openssl_name - rmdir $openssl_ssl/certs && ln -s /etc/ssl/certs $openssl_ssl/certs - # Download PyEnv from its GitHub repository. - export PYENV_ROOT=${{ env.PYENV_ROOT }} - export PATH=$PYENV_ROOT/bin:$PATH - git clone "https://github.com/pyenv/pyenv.git" "$PYENV_ROOT" - # Prevent pyenv build trying (and failing) to update pip - export GET_PIP=get-pip-2.6.py - echo 'import sys; sys.exit(0)' > ${GET_PIP} - GET_PIP=$(realpath $GET_PIP) - # Build and install Python - export CFLAGS="-I$openssl_inc" - export LDFLAGS="-L$openssl_lib" - export LD_LIBRARY_PATH="$openssl_lib" - pyenv install 2.6.9 - - name: Locate Python 2.6 - if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' }} - shell: bash - run: | - PYTHONHOME="${{ env.PYENV_ROOT }}/versions/2.6.9" - echo "PYTHONHOME=$PYTHONHOME" >> "$GITHUB_ENV" - echo "PATH=${PYTHONHOME}/bin:$PATH" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${{ env.openssl_dir }}/lib${LD_LIBRARY_PATH:+:}${LD_LIBRARY_PATH}" >> "$GITHUB_ENV" - #-------- Jython ------ - name: Set up Java 8 if: ${{ matrix.python-impl == 'jython' }} - uses: actions/setup-java@v3 + uses: actions/setup-java@v1 with: java-version: 8 - distribution: 'zulu' - - name: Setup Jython environment - if: ${{ matrix.python-impl == 'jython' }} - shell: bash - run: | - echo "JYTHON_ROOT=${HOME}/jython" >> "$GITHUB_ENV" - echo "PIP=pip" >> "$GITHUB_ENV" - - name: Cache Jython - id: cachejy - if: ${{ matrix.python-impl == 'jython' && matrix.python-version == '2.7' }} - uses: actions/cache@v3 - with: - # 2.7.3 now available, may solve SNI issue - key: jython-2.7.1 - path: | - ${{ env.JYTHON_ROOT }} - name: Install Jython - if: ${{ matrix.python-impl == 'jython' && matrix.python-version == '2.7' && ! steps.cachejy.outputs.cache-hit }} - shell: bash + if: ${{ matrix.python-impl == 'jython' }} run: | - JYTHON_ROOT="${{ env.JYTHON_ROOT }}" - curl -L "https://repo1.maven.org/maven2/org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar" -o jython-installer.jar - java -jar jython-installer.jar -s -d "${JYTHON_ROOT}" - echo "${JYTHON_ROOT}/bin" >> "$GITHUB_PATH" - - name: Set up cached Jython - if: ${{ steps.cachejy.outputs.cache-hit }} - shell: bash - run: | - JYTHON_ROOT="${{ env.JYTHON_ROOT }}" - echo "${JYTHON_ROOT}/bin" >> $GITHUB_PATH - - name: Install supporting Python 2.7 if possible - if: ${{ steps.cachejy.outputs.cache-hit }} - shell: bash - run: | - sudo apt-get install -y python2.7 || true - #-------- pip --------- - - name: Set up supported Python ${{ matrix.python-version }} pip - if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.7' }} - # This step may run in either Linux or Windows - shell: bash - run: | - echo "$PATH" - echo "$PYTHONHOME" - # curl is available on both Windows and Linux, -L follows redirects, -O gets name - python -m ensurepip || python -m pip --version || { \ - get_pip="${{ contains(needs.select.outputs.own-pip-versions, matrix.python-version) && format('{0}/', matrix.python-version) || '' }}"; \ - curl -L -O "https://bootstrap.pypa.io/pip/${get_pip}get-pip.py"; \ - python get-pip.py; } - - name: Set up Python 2.6 pip - if: ${{ matrix.python-version == '2.6' }} - shell: bash - run: | - python -m pip --version || { \ - curl -L -O "https://bootstrap.pypa.io/pip/2.6/get-pip.py"; \ - curl -L -O "https://files.pythonhosted.org/packages/ac/95/a05b56bb975efa78d3557efa36acaf9cf5d2fd0ee0062060493687432e03/pip-9.0.3-py2.py3-none-any.whl"; \ - python get-pip.py --no-setuptools --no-wheel pip-9.0.3-py2.py3-none-any.whl; } - # work-around to invoke pip module on 2.6: https://bugs.python.org/issue2751 - echo "PIP=python -m pip.__main__" >> "$GITHUB_ENV" - - name: Set up other Python ${{ matrix.python-version }} pip - if: ${{ matrix.python-version == '3.2' && steps.setup-python.outputs.python-path }} - shell: bash - run: | - python -m pip --version || { \ - curl -L -O "https://bootstrap.pypa.io/pip/3.2/get-pip.py"; \ - curl -L -O "https://files.pythonhosted.org/packages/b2/d0/cd115fe345dd6f07ec1c780020a7dfe74966fceeb171e0f20d1d4905b0b7/pip-7.1.2-py2.py3-none-any.whl"; \ - python get-pip.py --no-setuptools --no-wheel pip-7.1.2-py2.py3-none-any.whl; } - #-------- unittest ---- - - name: Upgrade Unittest for Python 2.6 - if: ${{ matrix.python-version == '2.6' }} - shell: bash - run: | - # Work around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb) - $PIP -qq show unittest2 || { \ - for u in "65/26/32b8464df2a97e6dd1b656ed26b2c194606c16fe163c695a992b36c11cdf/six-1.13.0-py2.py3-none-any.whl" \ - "f2/94/3af39d34be01a24a6e65433d19e107099374224905f1e0cc6bbe1fd22a2f/argparse-1.4.0-py2.py3-none-any.whl" \ - "c7/a3/c5da2a44c85bfbb6eebcfc1dde24933f8704441b98fdde6528f4831757a6/linecache2-1.0.0-py2.py3-none-any.whl" \ - "17/0a/6ac05a3723017a967193456a2efa0aa9ac4b51456891af1e2353bb9de21e/traceback2-1.4.0-py2.py3-none-any.whl" \ - "72/20/7f0f433060a962200b7272b8c12ba90ef5b903e218174301d0abfd523813/unittest2-1.1.0-py2.py3-none-any.whl"; do \ - curl -L -O "https://files.pythonhosted.org/packages/${u}"; \ - $PIP install ${u##*/}; \ - done; } - # make tests use unittest2 - for test in ./test/test_*.py ./test/helper.py; do - sed -r -i -e '/^import unittest$/s/test/test2 as unittest/' "$test" - done - #-------- nose -------- - - name: Install nose for Python ${{ matrix.python-version }} - if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || (matrix.python-impl == 'cpython' && (matrix.python-version == '2.7' || matrix.python-version == env.next)) }} - shell: bash - run: | - echo "$PATH" - echo "$PYTHONHOME" - # Use PyNose for recent Pythons instead of Nose - py3ver="${{ matrix.python-version }}" - py3ver=${py3ver#3.} - [ "$py3ver" != "${{ matrix.python-version }}" ] && py3ver=${py3ver%.*} || py3ver=0 - [ "$py3ver" -ge 9 ] && nose=pynose || nose=nose - $PIP -qq show $nose || $PIP install $nose - - name: Install nose for other Python 2 - if: ${{ matrix.python-impl == 'jython' || (matrix.python-impl == 'cpython' && matrix.python-version == '2.6') }} - shell: bash - run: | - # Work around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb) - $PIP -qq show nose || { \ - curl -L -O "https://files.pythonhosted.org/packages/99/4f/13fb671119e65c4dce97c60e67d3fd9e6f7f809f2b307e2611f4701205cb/nose-1.3.7-py2-none-any.whl"; \ - $PIP install nose-1.3.7-py2-none-any.whl; } - - name: Install nose for other Python 3 - if: ${{ matrix.python-version == '3.2' && steps.setup-python.outputs.python-path }} - shell: bash - run: | - $PIP -qq show nose || { \ - curl -L -O "https://files.pythonhosted.org/packages/15/d8/dd071918c040f50fa1cf80da16423af51ff8ce4a0f2399b7bf8de45ac3d9/nose-1.3.7-py3-none-any.whl"; \ - $PIP install nose-1.3.7-py3-none-any.whl; } - - name: Set up nosetest test - if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }} - shell: bash - run: | - # set PYTHON_VER - PYTHON_VER=${{ matrix.python-version }} - [ "${PYTHON_VER#*-}" != "$PYTHON_VER" ] || PYTHON_VER="${{ matrix.python-impl }}-${PYTHON_VER}" - echo "PYTHON_VER=$PYTHON_VER" >> "$GITHUB_ENV" - echo "PYTHON_IMPL=${{ matrix.python-impl }}" >> "$GITHUB_ENV" - # define a test to validate the Python version used by nosetests - printf '%s\n' \ - 'from __future__ import unicode_literals' \ - 'import sys, os, platform' \ - 'try:' \ - ' import unittest2 as unittest' \ - 'except ImportError:' \ - ' import unittest' \ - 'class TestPython(unittest.TestCase):' \ - ' def setUp(self):' \ - ' self.ver = os.environ["PYTHON_VER"].split("-")' \ - ' def test_python_ver(self):' \ - ' self.assertEqual(["%d" % v for v in sys.version_info[:2]], self.ver[-1].split(".")[:2])' \ - ' self.assertTrue(sys.version.startswith(self.ver[-1]))' \ - ' self.assertIn(self.ver[0], ",".join((sys.version, platform.python_implementation())).lower())' \ - ' def test_python_impl(self):' \ - ' self.assertIn(platform.python_implementation().lower(), (os.environ["PYTHON_IMPL"], self.ver[0]))' \ - > test/test_python.py - #-------- TESTS ------- + wget http://search.maven.org/remotecontent?filepath=org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar -O jython-installer.jar + java -jar jython-installer.jar -s -d "$HOME/jython" + echo "$HOME/jython/bin" >> $GITHUB_PATH + - name: Install nose + run: pip install nose - name: Run tests - if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }} continue-on-error: ${{ matrix.ytdl-test-set == 'download' || matrix.python-impl == 'jython' }} env: YTDL_TEST_SET: ${{ matrix.ytdl-test-set }} - run: | - ./devscripts/run_tests.${{ matrix.run-tests-ext }} + run: ./devscripts/run_tests.${{ matrix.run-tests-ext }} flake8: name: Linter runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v2 - name: Set up Python - uses: actions/setup-python@v4 + uses: actions/setup-python@v2 with: python-version: 3.9 - name: Install flake8 run: pip install flake8 - name: Run flake8 run: flake8 . - diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ff40cef78..58ab3a4b8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -150,7 +150,7 @@ After you have ensured this site is distributing its content legally, you can fo # TODO more properties (see youtube_dl/extractor/common.py) } ``` -5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py). This makes the extractor available for use, as long as the class ends with `IE`. +5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py). 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in. 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want. 8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart): diff --git a/ChangeLog b/ChangeLog index 658864282..22b4fa67d 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,115 +1,3 @@ -version 2021.12.17 - -Core -* [postprocessor/ffmpeg] Show ffmpeg output on error (#22680, #29336) - -Extractors -* [youtube] Update signature function patterns (#30363, #30366) -* [peertube] Only call description endpoint if necessary (#29383) -* [periscope] Pass referer to HLS requests (#29419) -- [liveleak] Remove extractor (#17625, #24222, #29331) -+ [pornhub] Add support for pornhubthbh7ap3u.onion -* [pornhub] Detect geo restriction -* [pornhub] Dismiss tbr extracted from download URLs (#28927) -* [curiositystream:collection] Extend _VALID_URL (#26326, #29117) -* [youtube] Make get_video_info processing more robust (#29333) -* [youtube] Workaround for get_video_info request (#29333) -* [bilibili] Strip uploader name (#29202) -* [youtube] Update invidious instance list (#29281) -* [umg:de] Update GraphQL API URL (#29304) -* [nrk] Switch psapi URL to https (#29344) -+ [egghead] Add support for app.egghead.io (#28404, #29303) -* [appleconnect] Fix extraction (#29208) -+ [orf:tvthek] Add support for MPD formats (#28672, #29236) - - -version 2021.06.06 - -Extractors -* [facebook] Improve login required detection -* [youporn] Fix formats and view count extraction (#29216) -* [orf:tvthek] Fix thumbnails extraction (#29217) -* [formula1] Fix extraction (#29206) -* [ard] Relax URL regular expression and fix video ids (#22724, #29091) -+ [ustream] Detect https embeds (#29133) -* [ted] Prefer own formats over external sources (#29142) -* [twitch:clips] Improve extraction (#29149) -+ [twitch:clips] Add access token query to download URLs (#29136) -* [youtube] Fix get_video_info request (#29086, #29165) -* [vimeo] Fix vimeo pro embed extraction (#29126) -* [redbulltv] Fix embed data extraction (#28770) -* [shahid] Relax URL regular expression (#28772, #28930) - - -version 2021.05.16 - -Core -* [options] Fix thumbnail option group name (#29042) -* [YoutubeDL] Improve extract_info doc (#28946) - -Extractors -+ [playstuff] Add support for play.stuff.co.nz (#28901, #28931) -* [eroprofile] Fix extraction (#23200, #23626, #29008) -+ [vivo] Add support for vivo.st (#29009) -+ [generic] Add support for og:audio (#28311, #29015) -* [phoenix] Fix extraction (#29057) -+ [generic] Add support for sibnet embeds -+ [vk] Add support for sibnet embeds (#9500) -+ [generic] Add Referer header for direct videojs download URLs (#2879, - #20217, #29053) -* [orf:radio] Switch download URLs to HTTPS (#29012, #29046) -- [blinkx] Remove extractor (#28941) -* [medaltv] Relax URL regular expression (#28884) -+ [funimation] Add support for optional lang code in URLs (#28950) -+ [gdcvault] Add support for HTML5 videos -* [dispeak] Improve FLV extraction (#13513, #28970) -* [kaltura] Improve iframe extraction (#28969) -* [kaltura] Make embed code alternatives actually work -* [cda] Improve extraction (#28709, #28937) -* [twitter] Improve formats extraction from vmap URL (#28909) -* [xtube] Fix formats extraction (#28870) -* [svtplay] Improve extraction (#28507, #28876) -* [tv2dk] Fix extraction (#28888) - - -version 2021.04.26 - -Extractors -+ [xfileshare] Add support for wolfstream.tv (#28858) -* [francetvinfo] Improve video id extraction (#28792) -* [medaltv] Fix extraction (#28807) -* [tver] Redirect all downloads to Brightcove (#28849) -* [go] Improve video id extraction (#25207, #25216, #26058) -* [youtube] Fix lazy extractors (#28780) -+ [bbc] Extract description and timestamp from __INITIAL_DATA__ (#28774) -* [cbsnews] Fix extraction for python <3.6 (#23359) - - -version 2021.04.17 - -Core -+ [utils] Add support for experimental HTTP response status code - 308 Permanent Redirect (#27877, #28768) - -Extractors -+ [lbry] Add support for HLS videos (#27877, #28768) -* [youtube] Fix stretched ratio calculation -* [youtube] Improve stretch extraction (#28769) -* [youtube:tab] Improve grid extraction (#28725) -+ [youtube:tab] Detect series playlist on playlists page (#28723) -+ [youtube] Add more invidious instances (#28706) -* [pluralsight] Extend anti-throttling timeout (#28712) -* [youtube] Improve URL to extractor routing (#27572, #28335, #28742) -+ [maoritv] Add support for maoritelevision.com (#24552) -+ [youtube:tab] Pass innertube context and x-goog-visitor-id header along with - continuation requests (#28702) -* [mtv] Fix Viacom A/B Testing Video Player extraction (#28703) -+ [pornhub] Extract DASH and HLS formats from get_media end point (#28698) -* [cbssports] Fix extraction (#28682) -* [jamendo] Fix track extraction (#28686) -* [curiositystream] Fix format extraction (#26845, #28668) - - version 2021.04.07 Core diff --git a/README.md b/README.md index 47e686f84..94c34d89a 100644 --- a/README.md +++ b/README.md @@ -33,7 +33,7 @@ Windows users can [download an .exe file](https://yt-dl.org/latest/youtube-dl.ex You can also use pip: sudo -H pip install --upgrade youtube-dl - + This command will update youtube-dl if you have already installed it. See the [pypi page](https://pypi.python.org/pypi/youtube_dl) for more information. macOS users can install youtube-dl with [Homebrew](https://brew.sh/): @@ -287,7 +287,7 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo --no-cache-dir Disable filesystem caching --rm-cache-dir Delete all filesystem cache files -## Thumbnail Options: +## Thumbnail images: --write-thumbnail Write thumbnail image to disk --write-all-thumbnails Write all thumbnail image formats to disk @@ -563,7 +563,7 @@ The basic usage is not to set any template arguments when downloading a single f - `is_live` (boolean): Whether this video is a live stream or a fixed-length video - `start_time` (numeric): Time in seconds where the reproduction should start, as specified in the URL - `end_time` (numeric): Time in seconds where the reproduction should end, as specified in the URL - - `format` (string): A human-readable description of the format + - `format` (string): A human-readable description of the format - `format_id` (string): Format code specified by `--format` - `format_note` (string): Additional info about the format - `width` (numeric): Width of the video @@ -632,7 +632,7 @@ To use percent literals in an output template use `%%`. To output to stdout use The current default template is `%(title)s-%(id)s.%(ext)s`. -In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title. +In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title: #### Output template and Windows batch files @@ -675,7 +675,7 @@ The general syntax for format selection is `--format FORMAT` or shorter `-f FORM **tl;dr:** [navigate me to examples](#format-selection-examples). -The simplest case is requesting a specific format, for example with `-f 22` you can download the format with format code equal to 22. You can get the list of available format codes for particular video using `--list-formats` or `-F`. Note that these format codes are extractor specific. +The simplest case is requesting a specific format, for example with `-f 22` you can download the format with format code equal to 22. You can get the list of available format codes for particular video using `--list-formats` or `-F`. Note that these format codes are extractor specific. You can also use a file extension (currently `3gp`, `aac`, `flv`, `m4a`, `mp3`, `mp4`, `ogg`, `wav`, `webm` are supported) to download the best quality format of a particular file extension served as a single file, e.g. `-f webm` will download the best quality format with the `webm` extension served as a single file. @@ -760,7 +760,7 @@ Videos can be filtered by their upload date using the options `--date`, `--dateb - Absolute dates: Dates in the format `YYYYMMDD`. - Relative dates: Dates in the format `(now|today)[+-][0-9](day|week|month|year)(s)?` - + Examples: ```bash @@ -893,7 +893,7 @@ Since June 2012 ([#342](https://github.com/ytdl-org/youtube-dl/issues/342)) yout ### The exe throws an error due to missing `MSVCR100.dll` -To run the exe you need to install first the [Microsoft Visual C++ 2010 Service Pack 1 Redistributable Package (x86)](https://download.microsoft.com/download/1/6/5/165255E7-1014-4D0A-B094-B6A430A6BFFC/vcredist_x86.exe). +To run the exe you need to install first the [Microsoft Visual C++ 2010 Redistributable Package (x86)](https://www.microsoft.com/en-US/download/details.aspx?id=5555). ### On Windows, how should I set up ffmpeg and youtube-dl? Where should I put the exe files? @@ -918,7 +918,7 @@ Either prepend `https://www.youtube.com/watch?v=` or separate the ID from the op Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. -In order to extract cookies from browser use any conforming browser extension for exporting cookies. For example, [Get cookies.txt LOCALLY](https://chrome.google.com/webstore/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc) (for Chrome) or [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) (for Firefox). +In order to extract cookies from browser use any conforming browser extension for exporting cookies. For example, [Get cookies.txt](https://chrome.google.com/webstore/detail/get-cookiestxt/bgaddhkoddajcdgocldbbfleckgcbcid/) (for Chrome) or [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) (for Firefox). Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows and `LF` (`\n`) for Unix and Unix-like systems (Linux, macOS, etc.). `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format. @@ -1000,8 +1000,6 @@ To run the test, simply invoke your favorite test runner, or execute a test file python test/test_download.py nosetests -For Python versions 3.6 and later, you can use [pynose](https://pypi.org/project/pynose/) to implement `nosetests`. The original [nose](https://pypi.org/project/nose/) has not been upgraded for 3.10 and later. - See item 6 of [new extractor tutorial](#adding-support-for-a-new-site) for how to run extractor specific test cases. If you want to create a build of youtube-dl yourself, you'll need @@ -1071,11 +1069,9 @@ After you have ensured this site is distributing its content legally, you can fo } ``` 5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py). -6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test (actually, test case) then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note: - * the test names use the extractor class name **without the trailing `IE`** - * tests with `only_matching` key in test's dict are not counted. -8. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want. -9. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart): +6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in. +7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want. +8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart): $ flake8 youtube_dl/extractor/yourextractor.py @@ -1093,7 +1089,7 @@ In any case, thank you very much for your contributions! ## youtube-dl coding conventions -This section introduces guidelines for writing idiomatic, robust and future-proof extractor code. +This section introduces a guide lines for writing idiomatic, robust and future-proof extractor code. Extractors are very fragile by nature since they depend on the layout of the source data provided by 3rd party media hosters out of your control and this layout tends to change. As an extractor implementer your task is not only to write code that will extract media links and metadata correctly but also to minimize dependency on the source's layout and even to make the code foresee potential future changes and be ready for that. This is important because it will allow the extractor not to break on minor layout changes thus keeping old youtube-dl versions working. Even though this breakage issue is easily fixed by emitting a new version of youtube-dl with a fix incorporated, all the previous versions become broken in all repositories and distros' packages that may not be so prompt in fetching the update from us. Needless to say, some non rolling release distros may never receive an update at all. @@ -1116,7 +1112,7 @@ Say you have some source dictionary `meta` that you've fetched as JSON with HTTP ```python meta = self._download_json(url, video_id) ``` - + Assume at this point `meta`'s layout is: ```python @@ -1160,7 +1156,7 @@ description = self._search_regex( ``` On failure this code will silently continue the extraction with `description` set to `None`. That is useful for metafields that may or may not be present. - + ### Provide fallbacks When extracting metadata try to do so from multiple sources. For example if `title` is present in several places, try extracting from at least some of them. This makes it more future-proof in case some of the sources become unavailable. @@ -1208,7 +1204,7 @@ r'(id|ID)=(?P\d+)' #### Make regular expressions relaxed and flexible When using regular expressions try to write them fuzzy, relaxed and flexible, skipping insignificant parts that are more likely to change, allowing both single and double quotes for quoted values and so on. - + ##### Example Say you need to extract `title` from the following HTML code: @@ -1232,7 +1228,7 @@ title = self._search_regex( webpage, 'title', group='title') ``` -Note how you tolerate potential changes in the `style` attribute's value or switch from using double quotes to single for `class` attribute: +Note how you tolerate potential changes in the `style` attribute's value or switch from using double quotes to single for `class` attribute: The code definitely should not look like: @@ -1333,114 +1329,27 @@ Wrap all extracted numeric data into safe functions from [`youtube_dl/utils.py`] Use `url_or_none` for safe URL processing. -Use `traverse_obj` for safe metadata extraction from parsed JSON. +Use `try_get` for safe metadata extraction from parsed JSON. -Use `unified_strdate` for uniform `upload_date` or any `YYYYMMDD` meta field extraction, `unified_timestamp` for uniform `timestamp` extraction, `parse_filesize` for `filesize` extraction, `parse_count` for count meta fields extraction, `parse_resolution`, `parse_duration` for `duration` extraction, `parse_age_limit` for `age_limit` extraction. +Use `unified_strdate` for uniform `upload_date` or any `YYYYMMDD` meta field extraction, `unified_timestamp` for uniform `timestamp` extraction, `parse_filesize` for `filesize` extraction, `parse_count` for count meta fields extraction, `parse_resolution`, `parse_duration` for `duration` extraction, `parse_age_limit` for `age_limit` extraction. Explore [`youtube_dl/utils.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/utils.py) for more useful convenience functions. #### More examples ##### Safely extract optional description from parsed JSON - -When processing complex JSON, as often returned by site API requests or stashed in web pages for "hydration", you can use the `traverse_obj()` utility function to handle multiple fallback values and to ensure the expected type of metadata items. The function's docstring defines how the function works: also review usage in the codebase for more examples. - -In this example, a text `description`, or `None`, is pulled from the `.result.video[0].summary` member of the parsed JSON `response`, if available. - -```python -description = traverse_obj(response, ('result', 'video', 0, 'summary', T(compat_str))) -``` -`T(...)` is a shorthand for a set literal; if you hate people who still run Python 2.6, `T(type_or_transformation)` could be written as a set literal `{type_or_transformation}`. - -Some extractors use the older and less capable `try_get()` function in the same way. - ```python description = try_get(response, lambda x: x['result']['video'][0]['summary'], compat_str) ``` ##### Safely extract more optional metadata - -In this example, various optional metadata values are extracted from the `.result.video[0]` member of the parsed JSON `response`, which is expected to be a JS object, parsed into a `dict`, with no crash if that isn't so, or if any of the target values are missing or invalid. - ```python -video = traverse_obj(response, ('result', 'video', 0, T(dict))) or {} -# formerly: -# video = try_get(response, lambda x: x['result']['video'][0], dict) or {} +video = try_get(response, lambda x: x['result']['video'][0], dict) or {} description = video.get('summary') duration = float_or_none(video.get('durationMs'), scale=1000) view_count = int_or_none(video.get('views')) ``` -#### Safely extract nested lists - -Suppose you've extracted JSON like this into a Python data structure named `media_json` using, say, the `_download_json()` or `_parse_json()` methods of `InfoExtractor`: -```json -{ - "title": "Example video", - "comment": "try extracting this", - "media": [{ - "type": "bad", - "size": 320, - "url": "https://some.cdn.site/bad.mp4" - }, { - "type": "streaming", - "url": "https://some.cdn.site/hls.m3u8" - }, { - "type": "super", - "size": 1280, - "url": "https://some.cdn.site/good.webm" - }], - "moreStuff": "more values", - ... -} -``` - -Then extractor code like this can collect the various fields of the JSON: -```python -... -from ..utils import ( - determine_ext, - int_or_none, - T, - traverse_obj, - txt_or_none, - url_or_none, -) -... - ... - info_dict = {} - # extract title and description if valid and not empty - info_dict.update(traverse_obj(media_json, { - 'title': ('title', T(txt_or_none)), - 'description': ('comment', T(txt_or_none)), - })) - - # extract any recognisable media formats - fmts = [] - # traverse into "media" list, extract `dict`s with desired keys - for fmt in traverse_obj(media_json, ('media', Ellipsis, { - 'format_id': ('type', T(txt_or_none)), - 'url': ('url', T(url_or_none)), - 'width': ('size', T(int_or_none)), })): - # bad `fmt` values were `None` and removed - if 'url' not in fmt: - continue - fmt_url = fmt['url'] # known to be valid URL - ext = determine_ext(fmt_url) - if ext == 'm3u8': - fmts.extend(self._extract_m3u8_formats(fmt_url, video_id, 'mp4', fatal=False)) - else: - fmt['ext'] = ext - fmts.append(fmt) - - # sort, raise if no formats - self._sort_formats(fmts) - - info_dict['formats'] = fmts - ... -``` -The extractor raises an exception rather than random crashes if the JSON structure changes so that no formats are found. - # EMBEDDING YOUTUBE-DL youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/ytdl-org/youtube-dl/issues/new). @@ -1497,11 +1406,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl: # BUGS -Bugs and suggestions should be reported in the issue tracker: ( is an alias for this). Unless you were prompted to or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the IRC channel [#youtube-dl](irc://chat.freenode.net/#youtube-dl) on freenode ([webchat](https://webchat.freenode.net/?randomnick=1&channels=youtube-dl)). - -## Opening a bug report or suggestion - -Be sure to follow instructions provided **below** and **in the issue tracker**. Complete the appropriate issue template fully. Consider whether your problem is covered by an existing issue: if so, follow the discussion there. Avoid commenting on existing duplicate issues as such comments do not add to the discussion of the issue and are liable to be treated as spam. +Bugs and suggestions should be reported at: . Unless you were prompted to or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the IRC channel [#youtube-dl](irc://chat.freenode.net/#youtube-dl) on freenode ([webchat](https://webchat.freenode.net/?randomnick=1&channels=youtube-dl)). **Please include the full output of youtube-dl when run with `-v`**, i.e. **add** `-v` flag to **your command line**, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this: ``` @@ -1521,17 +1426,17 @@ $ youtube-dl -v The output (including the first lines) contains important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever. -Finally please review your issue to avoid various common mistakes (you can and should use this as a checklist) listed below. +Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist): ### Is the description of the issue itself sufficient? -We often get issue reports that are hard to understand. To avoid subsequent clarifications, and to assist participants who are not native English speakers, please elaborate on what feature you are requesting, or what bug you want to be fixed. +We often get issue reports that we cannot really decipher. While in most cases we eventually get the required information after asking back multiple times, this poses an unnecessary drain on our resources. Many contributors, including myself, are also not native speakers, so we may misread some parts. -Make sure that it's obvious +So please elaborate on what feature you are requesting, or what bug you want to be fixed. Make sure that it's obvious - What the problem is - How it could be fixed -- How your proposed solution would look +- How your proposed solution would look like If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a committer myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over. @@ -1541,14 +1446,14 @@ If your server has multiple IPs or you suspect censorship, adding `--call-home` **Site support requests must contain an example URL**. An example URL is a URL you might want to download, like `https://www.youtube.com/watch?v=BaW_jenozKc`. There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. `https://www.youtube.com/`) is *not* an example URL. -### Is the issue already documented? - -Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or browse the [GitHub Issues](https://github.com/ytdl-org/youtube-dl/search?type=Issues) of this repository. Initially, at least, use the search term `-label:duplicate` to focus on active issues. If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity. - ### Are you using the latest version? Before reporting any issue, type `youtube-dl -U`. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well. +### Is the issue already documented? + +Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or browse the [GitHub Issues](https://github.com/ytdl-org/youtube-dl/search?type=Issues) of this repository. If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity. + ### Why are existing options not enough? Before requesting a new feature, please have a quick peek at [the list of supported options](https://github.com/ytdl-org/youtube-dl/blob/master/README.md#options). Many feature requests are for features that actually exist already! Please, absolutely do show off your work in the issue report and detail how the existing similar options do *not* solve your problem. diff --git a/devscripts/__init__.py b/devscripts/__init__.py deleted file mode 100644 index 750dbdca7..000000000 --- a/devscripts/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Empty file needed to make devscripts.utils properly importable from outside diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py index 7db396a77..3d1391334 100755 --- a/devscripts/bash-completion.py +++ b/devscripts/bash-completion.py @@ -5,12 +5,8 @@ import os from os.path import dirname as dirn import sys -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) - +sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) import youtube_dl -from youtube_dl.compat import compat_open as open - -from utils import read_file BASH_COMPLETION_FILE = "youtube-dl.bash-completion" BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in" @@ -22,8 +18,9 @@ def build_completion(opt_parser): for option in group.option_list: # for every long flag opts_flag.append(option.get_opt_string()) - template = read_file(BASH_COMPLETION_TEMPLATE) - with open(BASH_COMPLETION_FILE, "w", encoding='utf-8') as f: + with open(BASH_COMPLETION_TEMPLATE) as f: + template = f.read() + with open(BASH_COMPLETION_FILE, "w") as f: # just using the special char filled_template = template.replace("{{flags}}", " ".join(opts_flag)) f.write(filled_template) diff --git a/devscripts/cli_to_api.py b/devscripts/cli_to_api.py deleted file mode 100755 index 9fb1d2ba8..000000000 --- a/devscripts/cli_to_api.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -from __future__ import unicode_literals - -""" -This script displays the API parameters corresponding to a yt-dl command line - -Example: -$ ./cli_to_api.py -f best -{u'format': 'best'} -$ -""" - -# Allow direct execution -import os -import sys -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -import youtube_dl -from types import MethodType - - -def cli_to_api(*opts): - YDL = youtube_dl.YoutubeDL - - # to extract the parsed options, break out of YoutubeDL instantiation - - # return options via this Exception - class ParseYTDLResult(Exception): - def __init__(self, result): - super(ParseYTDLResult, self).__init__('result') - self.opts = result - - # replacement constructor that raises ParseYTDLResult - def ytdl_init(ydl, ydl_opts): - super(YDL, ydl).__init__(ydl_opts) - raise ParseYTDLResult(ydl_opts) - - # patch in the constructor - YDL.__init__ = MethodType(ytdl_init, YDL) - - # core parser - def parsed_options(argv): - try: - youtube_dl._real_main(list(argv)) - except ParseYTDLResult as result: - return result.opts - - # from https://github.com/yt-dlp/yt-dlp/issues/5859#issuecomment-1363938900 - default = parsed_options([]) - - def neq_opt(a, b): - if a == b: - return False - if a is None and repr(type(object)).endswith(".utils.DateRange'>"): - return '0001-01-01 - 9999-12-31' != '{0}'.format(b) - return a != b - - diff = dict((k, v) for k, v in parsed_options(opts).items() if neq_opt(default[k], v)) - if 'postprocessors' in diff: - diff['postprocessors'] = [pp for pp in diff['postprocessors'] if pp not in default['postprocessors']] - return diff - - -def main(): - from pprint import PrettyPrinter - - pprint = PrettyPrinter() - super_format = pprint.format - - def format(object, context, maxlevels, level): - if repr(type(object)).endswith(".utils.DateRange'>"): - return '{0}: {1}>'.format(repr(object)[:-2], object), True, False - return super_format(object, context, maxlevels, level) - - pprint.format = format - - pprint.pprint(cli_to_api(*sys.argv)) - - -if __name__ == '__main__': - main() diff --git a/devscripts/create-github-release.py b/devscripts/create-github-release.py index 320bcfc27..2ddfa1096 100644 --- a/devscripts/create-github-release.py +++ b/devscripts/create-github-release.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from __future__ import unicode_literals +import io import json import mimetypes import netrc @@ -9,9 +10,7 @@ import os import re import sys -dirn = os.path.dirname - -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from youtube_dl.compat import ( compat_basestring, @@ -23,7 +22,6 @@ from youtube_dl.utils import ( make_HTTPS_handler, sanitized_Request, ) -from utils import read_file class GitHubReleaser(object): @@ -91,7 +89,8 @@ def main(): changelog_file, version, build_path = args - changelog = read_file(changelog_file) + with io.open(changelog_file, encoding='utf-8') as inf: + changelog = inf.read() mobj = re.search(r'(?s)version %s\n{2}(.+?)\n{3}' % version, changelog) body = mobj.group(1) if mobj else '' diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py index ef8a39e0b..51d19dd33 100755 --- a/devscripts/fish-completion.py +++ b/devscripts/fish-completion.py @@ -6,13 +6,10 @@ import os from os.path import dirname as dirn import sys -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) - +sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) import youtube_dl from youtube_dl.utils import shell_quote -from utils import read_file, write_file - FISH_COMPLETION_FILE = 'youtube-dl.fish' FISH_COMPLETION_TEMPLATE = 'devscripts/fish-completion.in' @@ -41,9 +38,11 @@ def build_completion(opt_parser): complete_cmd.extend(EXTRA_ARGS.get(long_option, [])) commands.append(shell_quote(complete_cmd)) - template = read_file(FISH_COMPLETION_TEMPLATE) + with open(FISH_COMPLETION_TEMPLATE) as f: + template = f.read() filled_template = template.replace('{{commands}}', '\n'.join(commands)) - write_file(FISH_COMPLETION_FILE, filled_template) + with open(FISH_COMPLETION_FILE, 'w') as f: + f.write(filled_template) parser = youtube_dl.parseOpts()[0] diff --git a/devscripts/gh-pages/add-version.py b/devscripts/gh-pages/add-version.py index b84908f85..867ea0048 100755 --- a/devscripts/gh-pages/add-version.py +++ b/devscripts/gh-pages/add-version.py @@ -6,21 +6,16 @@ import sys import hashlib import os.path -dirn = os.path.dirname - -sys.path.insert(0, dirn(dirn(dirn(os.path.abspath(__file__))))) - -from devscripts.utils import read_file, write_file -from youtube_dl.compat import compat_open as open if len(sys.argv) <= 1: print('Specify the version number as parameter') sys.exit() version = sys.argv[1] -write_file('update/LATEST_VERSION', version) +with open('update/LATEST_VERSION', 'w') as f: + f.write(version) -versions_info = json.loads(read_file('update/versions.json')) +versions_info = json.load(open('update/versions.json')) if 'signature' in versions_info: del versions_info['signature'] @@ -44,5 +39,5 @@ for key, filename in filenames.items(): versions_info['versions'][version] = new_version versions_info['latest'] = version -with open('update/versions.json', 'w', encoding='utf-8') as jsonf: - json.dumps(versions_info, jsonf, indent=4, sort_keys=True) +with open('update/versions.json', 'w') as jsonf: + json.dump(versions_info, jsonf, indent=4, sort_keys=True) diff --git a/devscripts/gh-pages/generate-download.py b/devscripts/gh-pages/generate-download.py index 3e38e9299..a873d32ee 100755 --- a/devscripts/gh-pages/generate-download.py +++ b/devscripts/gh-pages/generate-download.py @@ -2,21 +2,14 @@ from __future__ import unicode_literals import json -import os.path -import sys -dirn = os.path.dirname - -sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) - -from utils import read_file, write_file - -versions_info = json.loads(read_file('update/versions.json')) +versions_info = json.load(open('update/versions.json')) version = versions_info['latest'] version_dict = versions_info['versions'][version] # Read template page -template = read_file('download.html.in') +with open('download.html.in', 'r', encoding='utf-8') as tmplf: + template = tmplf.read() template = template.replace('@PROGRAM_VERSION@', version) template = template.replace('@PROGRAM_URL@', version_dict['bin'][0]) @@ -25,5 +18,5 @@ template = template.replace('@EXE_URL@', version_dict['exe'][0]) template = template.replace('@EXE_SHA256SUM@', version_dict['exe'][1]) template = template.replace('@TAR_URL@', version_dict['tar'][0]) template = template.replace('@TAR_SHA256SUM@', version_dict['tar'][1]) - -write_file('download.html', template) +with open('download.html', 'w', encoding='utf-8') as dlf: + dlf.write(template) diff --git a/devscripts/gh-pages/update-copyright.py b/devscripts/gh-pages/update-copyright.py index 444595c48..61487f925 100755 --- a/devscripts/gh-pages/update-copyright.py +++ b/devscripts/gh-pages/update-copyright.py @@ -5,22 +5,17 @@ from __future__ import with_statement, unicode_literals import datetime import glob +import io # For Python 2 compatibility import os import re -import sys -dirn = os.path.dirname - -sys.path.insert(0, dirn(dirn(dirn(os.path.abspath(__file__))))) - -from devscripts.utils import read_file, write_file -from youtube_dl import compat_str - -year = compat_str(datetime.datetime.now().year) +year = str(datetime.datetime.now().year) for fn in glob.glob('*.html*'): - content = read_file(fn) + with io.open(fn, encoding='utf-8') as f: + content = f.read() newc = re.sub(r'(?PCopyright © 2011-)(?P[0-9]{4})', 'Copyright © 2011-' + year, content) if content != newc: tmpFn = fn + '.part' - write_file(tmpFn, newc) + with io.open(tmpFn, 'wt', encoding='utf-8') as outf: + outf.write(newc) os.rename(tmpFn, fn) diff --git a/devscripts/gh-pages/update-feed.py b/devscripts/gh-pages/update-feed.py index 13a367d34..506a62377 100755 --- a/devscripts/gh-pages/update-feed.py +++ b/devscripts/gh-pages/update-feed.py @@ -2,16 +2,10 @@ from __future__ import unicode_literals import datetime +import io import json -import os.path import textwrap -import sys -dirn = os.path.dirname - -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) - -from utils import write_file atom_template = textwrap.dedent("""\ @@ -78,4 +72,5 @@ for v in versions: entries_str = textwrap.indent(''.join(entries), '\t') atom_template = atom_template.replace('@ENTRIES@', entries_str) -write_file('update/releases.atom', atom_template) +with io.open('update/releases.atom', 'w', encoding='utf-8') as atom_file: + atom_file.write(atom_template) diff --git a/devscripts/gh-pages/update-sites.py b/devscripts/gh-pages/update-sites.py index 06a8a474c..531c93c70 100755 --- a/devscripts/gh-pages/update-sites.py +++ b/devscripts/gh-pages/update-sites.py @@ -5,17 +5,15 @@ import sys import os import textwrap -dirn = os.path.dirname - # We must be able to import youtube_dl -sys.path.insert(0, dirn(dirn(dirn(os.path.abspath(__file__))))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) import youtube_dl -from devscripts.utils import read_file, write_file def main(): - template = read_file('supportedsites.html.in') + with open('supportedsites.html.in', 'r', encoding='utf-8') as tmplf: + template = tmplf.read() ie_htmls = [] for ie in youtube_dl.list_extractors(age_limit=None): @@ -31,7 +29,8 @@ def main(): template = template.replace('@SITES@', textwrap.indent('\n'.join(ie_htmls), '\t')) - write_file('supportedsites.html', template) + with open('supportedsites.html', 'w', encoding='utf-8') as sitesf: + sitesf.write(template) if __name__ == '__main__': diff --git a/devscripts/make_contributing.py b/devscripts/make_contributing.py index 5a9eb194f..226d1a5d6 100755 --- a/devscripts/make_contributing.py +++ b/devscripts/make_contributing.py @@ -1,11 +1,10 @@ #!/usr/bin/env python from __future__ import unicode_literals +import io import optparse import re -from utils import read_file, write_file - def main(): parser = optparse.OptionParser(usage='%prog INFILE OUTFILE') @@ -15,7 +14,8 @@ def main(): infile, outfile = args - readme = read_file(infile) + with io.open(infile, encoding='utf-8') as inf: + readme = inf.read() bug_text = re.search( r'(?s)#\s*BUGS\s*[^\n]*\s*(.*?)#\s*COPYRIGHT', readme).group(1) @@ -25,7 +25,8 @@ def main(): out = bug_text + dev_text - write_file(outfile, out) + with io.open(outfile, 'w', encoding='utf-8') as outf: + outf.write(out) if __name__ == '__main__': diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py index 65fa8169f..b7ad23d83 100644 --- a/devscripts/make_issue_template.py +++ b/devscripts/make_issue_template.py @@ -1,11 +1,8 @@ #!/usr/bin/env python from __future__ import unicode_literals +import io import optparse -import os.path -import sys - -from utils import read_file, read_version, write_file def main(): @@ -16,11 +13,17 @@ def main(): infile, outfile = args - issue_template_tmpl = read_file(infile) + with io.open(infile, encoding='utf-8') as inf: + issue_template_tmpl = inf.read() - out = issue_template_tmpl % {'version': read_version()} + # Get the version from youtube_dl/version.py without importing the package + exec(compile(open('youtube_dl/version.py').read(), + 'youtube_dl/version.py', 'exec')) - write_file(outfile, out) + out = issue_template_tmpl % {'version': locals()['__version__']} + + with io.open(outfile, 'w', encoding='utf-8') as outf: + outf.write(out) if __name__ == '__main__': main() diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 5b8b123a4..878ae72b1 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -1,49 +1,28 @@ from __future__ import unicode_literals, print_function from inspect import getsource +import io import os from os.path import dirname as dirn -import re import sys print('WARNING: Lazy loading extractors is an experimental feature that may not always work', file=sys.stderr) -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) +sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) lazy_extractors_filename = sys.argv[1] if os.path.exists(lazy_extractors_filename): os.remove(lazy_extractors_filename) -# Py2: may be confused by leftover lazy_extractors.pyc -if sys.version_info[0] < 3: - for c in ('c', 'o'): - try: - os.remove(lazy_extractors_filename + 'c') - except OSError: - pass - -from devscripts.utils import read_file, write_file -from youtube_dl.compat import compat_register_utf8 - -compat_register_utf8() from youtube_dl.extractor import _ALL_CLASSES from youtube_dl.extractor.common import InfoExtractor, SearchInfoExtractor -module_template = read_file('devscripts/lazy_load_template.py') - - -def get_source(m): - return re.sub(r'(?m)^\s*#.*\n', '', getsource(m)) - +with open('devscripts/lazy_load_template.py', 'rt') as f: + module_template = f.read() module_contents = [ - module_template, - get_source(InfoExtractor.suitable), - get_source(InfoExtractor._match_valid_url) + '\n', - 'class LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n', - # needed for suitable() methods of Youtube extractor (see #28780) - 'from youtube_dl.utils import parse_qs, variadic\n', -] + module_template + '\n' + getsource(InfoExtractor.suitable) + '\n', + 'class LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n'] ie_template = ''' class {name}({bases}): @@ -75,7 +54,7 @@ def build_lazy_ie(ie, name): valid_url=valid_url, module=ie.__module__) if ie.suitable.__func__ is not InfoExtractor.suitable.__func__: - s += '\n' + get_source(ie.suitable) + s += '\n' + getsource(ie.suitable) if hasattr(ie, '_make_valid_url'): # search extractors s += make_valid_template.format(valid_url=ie._make_valid_url()) @@ -115,17 +94,7 @@ for ie in ordered_cls: module_contents.append( '_ALL_CLASSES = [{0}]'.format(', '.join(names))) -module_src = '\n'.join(module_contents) +module_src = '\n'.join(module_contents) + '\n' -write_file(lazy_extractors_filename, module_src + '\n') - -# work around JVM byte code module limit in Jython -if sys.platform.startswith('java') and sys.version_info[:2] == (2, 7): - import subprocess - from youtube_dl.compat import compat_subprocess_get_DEVNULL - # if Python 2.7 is available, use it to compile the module for Jython - try: - # if Python 2.7 is available, use it to compile the module for Jython - subprocess.check_call(['python2.7', '-m', 'py_compile', lazy_extractors_filename], stdout=compat_subprocess_get_DEVNULL()) - except Exception: - pass +with io.open(lazy_extractors_filename, 'wt', encoding='utf-8') as f: + f.write(module_src) diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py index 7a5b04dcc..8fbce0796 100755 --- a/devscripts/make_readme.py +++ b/devscripts/make_readme.py @@ -1,14 +1,8 @@ from __future__ import unicode_literals -import os.path -import re +import io import sys -dirn = os.path.dirname - -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) - -from utils import read_file -from youtube_dl.compat import compat_open as open +import re README_FILE = 'README.md' helptext = sys.stdin.read() @@ -16,7 +10,8 @@ helptext = sys.stdin.read() if isinstance(helptext, bytes): helptext = helptext.decode('utf-8') -oldreadme = read_file(README_FILE) +with io.open(README_FILE, encoding='utf-8') as f: + oldreadme = f.read() header = oldreadme[:oldreadme.index('# OPTIONS')] footer = oldreadme[oldreadme.index('# CONFIGURATION'):] @@ -25,7 +20,7 @@ options = helptext[helptext.index(' General Options:') + 19:] options = re.sub(r'(?m)^ (\w.+)$', r'## \1', options) options = '# OPTIONS\n' + options + '\n' -with open(README_FILE, 'w', encoding='utf-8') as f: +with io.open(README_FILE, 'w', encoding='utf-8') as f: f.write(header) f.write(options) f.write(footer) diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py index c424d18d7..764795bc5 100644 --- a/devscripts/make_supportedsites.py +++ b/devscripts/make_supportedsites.py @@ -1,19 +1,17 @@ #!/usr/bin/env python from __future__ import unicode_literals +import io import optparse -import os.path +import os import sys + # Import youtube_dl -dirn = os.path.dirname - -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) - +ROOT_DIR = os.path.join(os.path.dirname(__file__), '..') +sys.path.insert(0, ROOT_DIR) import youtube_dl -from utils import write_file - def main(): parser = optparse.OptionParser(usage='%prog OUTFILE.md') @@ -40,7 +38,8 @@ def main(): ' - ' + md + '\n' for md in gen_ies_md(ies)) - write_file(outfile, out) + with io.open(outfile, 'w', encoding='utf-8') as outf: + outf.write(out) if __name__ == '__main__': diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py index 0090ada3e..76bf873e1 100644 --- a/devscripts/prepare_manpage.py +++ b/devscripts/prepare_manpage.py @@ -1,13 +1,13 @@ from __future__ import unicode_literals +import io import optparse import os.path import re -from utils import read_file, write_file - ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) README_FILE = os.path.join(ROOT_DIR, 'README.md') + PREFIX = r'''%YOUTUBE-DL(1) # NAME @@ -29,7 +29,8 @@ def main(): outfile, = args - readme = read_file(README_FILE) + with io.open(README_FILE, encoding='utf-8') as f: + readme = f.read() readme = re.sub(r'(?s)^.*?(?=# DESCRIPTION)', '', readme) readme = re.sub(r'\s+youtube-dl \[OPTIONS\] URL \[URL\.\.\.\]', '', readme) @@ -37,7 +38,8 @@ def main(): readme = filter_options(readme) - write_file(outfile, readme) + with io.open(outfile, 'w', encoding='utf-8') as outf: + outf.write(readme) def filter_options(readme): diff --git a/devscripts/utils.py b/devscripts/utils.py deleted file mode 100644 index 2d072d2e0..000000000 --- a/devscripts/utils.py +++ /dev/null @@ -1,62 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -import argparse -import functools -import os.path -import subprocess -import sys - -dirn = os.path.dirname - -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) - -from youtube_dl.compat import ( - compat_kwargs, - compat_open as open, -) - - -def read_file(fname): - with open(fname, encoding='utf-8') as f: - return f.read() - - -def write_file(fname, content, mode='w'): - with open(fname, mode, encoding='utf-8') as f: - return f.write(content) - - -def read_version(fname='youtube_dl/version.py'): - """Get the version without importing the package""" - exec(compile(read_file(fname), fname, 'exec')) - return locals()['__version__'] - - -def get_filename_args(has_infile=False, default_outfile=None): - parser = argparse.ArgumentParser() - if has_infile: - parser.add_argument('infile', help='Input file') - kwargs = {'nargs': '?', 'default': default_outfile} if default_outfile else {} - kwargs['help'] = 'Output file' - parser.add_argument('outfile', **compat_kwargs(kwargs)) - - opts = parser.parse_args() - if has_infile: - return opts.infile, opts.outfile - return opts.outfile - - -def compose_functions(*functions): - return lambda x: functools.reduce(lambda y, f: f(y), functions, x) - - -def run_process(*args, **kwargs): - kwargs.setdefault('text', True) - kwargs.setdefault('check', True) - kwargs.setdefault('capture_output', True) - if kwargs['text']: - kwargs.setdefault('encoding', 'utf-8') - kwargs.setdefault('errors', 'replace') - kwargs = compat_kwargs(kwargs) - return subprocess.run(args, **kwargs) diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py index ebd552fcb..60aaf76cc 100755 --- a/devscripts/zsh-completion.py +++ b/devscripts/zsh-completion.py @@ -7,8 +7,6 @@ import sys sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) import youtube_dl -from utils import read_file, write_file - ZSH_COMPLETION_FILE = "youtube-dl.zsh" ZSH_COMPLETION_TEMPLATE = "devscripts/zsh-completion.in" @@ -36,13 +34,15 @@ def build_completion(opt_parser): flags = [opt.get_opt_string() for opt in opts] - template = read_file(ZSH_COMPLETION_TEMPLATE) + with open(ZSH_COMPLETION_TEMPLATE) as f: + template = f.read() template = template.replace("{{fileopts}}", "|".join(fileopts)) template = template.replace("{{diropts}}", "|".join(diropts)) template = template.replace("{{flags}}", " ".join(flags)) - write_file(ZSH_COMPLETION_FILE, template) + with open(ZSH_COMPLETION_FILE, "w") as f: + f.write(template) parser = youtube_dl.parseOpts()[0] diff --git a/docs/supportedsites.md b/docs/supportedsites.md index ae2a6b8b0..ff9177a2c 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -3,7 +3,6 @@ - **20min** - **220.ro** - **23video** - - **247sports** - **24video** - **3qsdn**: 3Q SDN - **3sat** @@ -119,6 +118,7 @@ - **BitChuteChannel** - **BleacherReport** - **BleacherReportCMS** + - **blinkx** - **Bloomberg** - **BokeCC** - **BongaCams** @@ -160,8 +160,7 @@ - **cbsnews**: CBS News - **cbsnews:embed** - **cbsnews:livevideo**: CBS News Live Videos - - **cbssports** - - **cbssports:embed** + - **CBSSports** - **CCMA** - **CCTV**: 央视网 - **CDA** @@ -472,6 +471,8 @@ - **LinuxAcademy** - **LiTV** - **LiveJournal** + - **LiveLeak** + - **LiveLeakEmbed** - **livestream** - **livestream:original** - **LnkGo** @@ -489,7 +490,6 @@ - **mangomolo:live** - **mangomolo:video** - **ManyVids** - - **MaoriTV** - **Markiza** - **MarkizaPage** - **massengeschmack.tv** @@ -710,7 +710,6 @@ - **play.fm** - **player.sky.it** - **PlayPlusTV** - - **PlayStuff** - **PlaysTV** - **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz - **Playvid** @@ -1160,7 +1159,7 @@ - **WWE** - **XBef** - **XboxClips** - - **XFileShare**: XFileShare based sites: Aparat, ClipWatching, GoUnlimited, GoVid, HolaVid, Streamty, TheVideoBee, Uqload, VidBom, vidlo, VidLocker, VidShare, VUp, WolfStream, XVideoSharing + - **XFileShare**: XFileShare based sites: Aparat, ClipWatching, GoUnlimited, GoVid, HolaVid, Streamty, TheVideoBee, Uqload, VidBom, vidlo, VidLocker, VidShare, VUp, XVideoSharing - **XHamster** - **XHamsterEmbed** - **XHamsterUser** diff --git a/test/helper.py b/test/helper.py index 6f2129eff..e62aab11e 100644 --- a/test/helper.py +++ b/test/helper.py @@ -1,24 +1,22 @@ from __future__ import unicode_literals import errno +import io import hashlib import json import os.path import re +import types import ssl import sys -import types -import unittest import youtube_dl.extractor from youtube_dl import YoutubeDL from youtube_dl.compat import ( - compat_open as open, compat_os_name, compat_str, ) from youtube_dl.utils import ( - IDENTITY, preferredencoding, write_string, ) @@ -29,10 +27,10 @@ def get_params(override=None): "parameters.json") LOCAL_PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "local_parameters.json") - with open(PARAMETERS_FILE, encoding='utf-8') as pf: + with io.open(PARAMETERS_FILE, encoding='utf-8') as pf: parameters = json.load(pf) if os.path.exists(LOCAL_PARAMETERS_FILE): - with open(LOCAL_PARAMETERS_FILE, encoding='utf-8') as pf: + with io.open(LOCAL_PARAMETERS_FILE, encoding='utf-8') as pf: parameters.update(json.load(pf)) if override: parameters.update(override) @@ -74,8 +72,7 @@ class FakeYDL(YoutubeDL): def to_screen(self, s, skip_eol=None): print(s) - def trouble(self, *args, **kwargs): - s = args[0] if len(args) > 0 else kwargs.get('message', 'Missing message') + def trouble(self, s, tb=None): raise Exception(s) def download(self, x): @@ -92,17 +89,6 @@ class FakeYDL(YoutubeDL): self.report_warning = types.MethodType(report_warning, self) -class FakeLogger(object): - def debug(self, msg): - pass - - def warning(self, msg): - pass - - def error(self, msg): - pass - - def gettestcases(include_onlymatching=False): for ie in youtube_dl.extractor.gen_extractors(): for tc in ie.get_testcases(include_onlymatching): @@ -142,12 +128,6 @@ def expect_value(self, got, expected, field): self.assertTrue( contains_str in got, 'field %s (value: %r) should contain %r' % (field, got, contains_str)) - elif isinstance(expected, compat_str) and re.match(r'lambda \w+:', expected): - fn = eval(expected) - suite = expected.split(':', 1)[1].strip() - self.assertTrue( - fn(got), - 'Expected field %s to meet condition %s, but value %r failed ' % (field, suite, got)) elif isinstance(expected, type): self.assertTrue( isinstance(got, expected), @@ -157,7 +137,7 @@ def expect_value(self, got, expected, field): elif isinstance(expected, list) and isinstance(got, list): self.assertEqual( len(expected), len(got), - 'Expected a list of length %d, but got a list of length %d for field %s' % ( + 'Expect a list of length %d, but got a list of length %d for field %s' % ( len(expected), len(got), field)) for index, (item_got, item_expected) in enumerate(zip(got, expected)): type_got = type(item_got) @@ -181,18 +161,18 @@ def expect_value(self, got, expected, field): op, _, expected_num = expected.partition(':') expected_num = int(expected_num) if op == 'mincount': - assert_func = self.assertGreaterEqual + assert_func = assertGreaterEqual msg_tmpl = 'Expected %d items in field %s, but only got %d' elif op == 'maxcount': - assert_func = self.assertLessEqual + assert_func = assertLessEqual msg_tmpl = 'Expected maximum %d items in field %s, but got %d' elif op == 'count': - assert_func = self.assertEqual + assert_func = assertEqual msg_tmpl = 'Expected exactly %d items in field %s, but got %d' else: assert False assert_func( - len(got), expected_num, + self, len(got), expected_num, msg_tmpl % (expected_num, field, len(got))) return self.assertEqual( @@ -262,6 +242,27 @@ def assertRegexpMatches(self, text, regexp, msg=None): self.assertTrue(m, msg) +def assertGreaterEqual(self, got, expected, msg=None): + if not (got >= expected): + if msg is None: + msg = '%r not greater than or equal to %r' % (got, expected) + self.assertTrue(got >= expected, msg) + + +def assertLessEqual(self, got, expected, msg=None): + if not (got <= expected): + if msg is None: + msg = '%r not less than or equal to %r' % (got, expected) + self.assertTrue(got <= expected, msg) + + +def assertEqual(self, got, expected, msg=None): + if not (got == expected): + if msg is None: + msg = '%r not equal to %r' % (got, expected) + self.assertTrue(got == expected, msg) + + def expect_warnings(ydl, warnings_re): real_warning = ydl.report_warning @@ -279,7 +280,3 @@ def http_server_port(httpd): else: sock = httpd.socket return sock.getsockname()[1] - - -def expectedFailureIf(cond): - return unittest.expectedFailure if cond else IDENTITY diff --git a/test/parameters.json b/test/parameters.json index 864c9d130..65fd54428 100644 --- a/test/parameters.json +++ b/test/parameters.json @@ -18,6 +18,7 @@ "noprogress": false, "outtmpl": "%(id)s.%(ext)s", "password": null, + "playlistend": -1, "playliststart": 1, "prefer_free_formats": false, "quiet": false, diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py index 09100a1d6..dd69a681b 100644 --- a/test/test_InfoExtractor.py +++ b/test/test_InfoExtractor.py @@ -3,36 +3,18 @@ from __future__ import unicode_literals # Allow direct execution +import io import os import sys import unittest - sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import threading - -from test.helper import ( - expect_dict, - expect_value, - FakeYDL, - http_server_port, -) -from youtube_dl.compat import ( - compat_etree_fromstring, - compat_http_server, - compat_open as open, -) +from test.helper import FakeYDL, expect_dict, expect_value, http_server_port +from youtube_dl.compat import compat_etree_fromstring, compat_http_server from youtube_dl.extractor.common import InfoExtractor -from youtube_dl.extractor import ( - get_info_extractor, - YoutubeIE, -) -from youtube_dl.utils import ( - encode_data_uri, - ExtractorError, - RegexNotFoundError, - strip_jsonp, -) +from youtube_dl.extractor import YoutubeIE, get_info_extractor +from youtube_dl.utils import encode_data_uri, strip_jsonp, ExtractorError, RegexNotFoundError +import threading TEAPOT_RESPONSE_STATUS = 418 @@ -53,13 +35,13 @@ class InfoExtractorTestRequestHandler(compat_http_server.BaseHTTPRequestHandler) assert False -class DummyIE(InfoExtractor): +class TestIE(InfoExtractor): pass class TestInfoExtractor(unittest.TestCase): def setUp(self): - self.ie = DummyIE(FakeYDL()) + self.ie = TestIE(FakeYDL()) def test_ie_key(self): self.assertEqual(get_info_extractor(YoutubeIE.ie_key()), YoutubeIE) @@ -80,7 +62,6 @@ class TestInfoExtractor(unittest.TestCase): - ''' self.assertEqual(ie._og_search_title(html), 'Foo') self.assertEqual(ie._og_search_description(html), 'Some video\'s description ') @@ -93,7 +74,6 @@ class TestInfoExtractor(unittest.TestCase): self.assertEqual(ie._og_search_property(('test0', 'test1'), html), 'foo > < bar') self.assertRaises(RegexNotFoundError, ie._og_search_property, 'test0', html, None, fatal=True) self.assertRaises(RegexNotFoundError, ie._og_search_property, ('test0', 'test00'), html, None, fatal=True) - self.assertEqual(ie._og_search_property('test4', html), 'unquoted-value') def test_html_search_meta(self): ie = self.ie @@ -118,74 +98,6 @@ class TestInfoExtractor(unittest.TestCase): self.assertRaises(RegexNotFoundError, ie._html_search_meta, 'z', html, None, fatal=True) self.assertRaises(RegexNotFoundError, ie._html_search_meta, ('z', 'x'), html, None, fatal=True) - def test_search_nextjs_data(self): - html = ''' - - - - - - Test _search_nextjs_data() - - -
-
-
-
-
-
-
-
-
-
-
-
-
-
-
- - - -''' - search = self.ie._search_nextjs_data(html, 'testID') - self.assertEqual(search['props']['pageProps']['video']['id'], 'testid') - search = self.ie._search_nextjs_data( - 'no next.js data here, move along', 'testID', default={'status': 0}) - self.assertEqual(search['status'], 0) - - def test_search_nuxt_data(self): - html = ''' - - - - - Nuxt.js Test Page - - - - -
-

Example heading

-
-

Decoy text

-
-
- - - - -''' - search = self.ie._search_nuxt_data(html, 'testID') - self.assertEqual(search['track']['id'], 'testid') - def test_search_json_ld_realworld(self): # https://github.com/ytdl-org/youtube-dl/issues/23306 expect_dict( @@ -434,24 +346,6 @@ class TestInfoExtractor(unittest.TestCase): }], }) - # from https://0000.studio/ - # with type attribute but without extension in URL - expect_dict( - self, - self.ie._parse_html5_media_entries( - 'https://0000.studio', - r''' - - ''', None)[0], - { - 'formats': [{ - 'url': 'https://d1ggyt9m8pwf3g.cloudfront.net/protected/ap-northeast-1:1864af40-28d5-492b-b739-b32314b1a527/archive/clip/838db6a7-8973-4cd6-840d-8517e4093c92', - 'ext': 'mp4', - }], - }) - def test_extract_jwplayer_data_realworld(self): # from http://www.suffolk.edu/sjc/ expect_dict( @@ -905,8 +799,8 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/ ] for m3u8_file, m3u8_url, expected_formats in _TEST_CASES: - with open('./test/testdata/m3u8/%s.m3u8' % m3u8_file, - mode='r', encoding='utf-8') as f: + with io.open('./test/testdata/m3u8/%s.m3u8' % m3u8_file, + mode='r', encoding='utf-8') as f: formats = self.ie._parse_m3u8_formats( f.read(), m3u8_url, ext='mp4') self.ie._sort_formats(formats) @@ -996,8 +890,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/ 'tbr': 5997.485, 'width': 1920, 'height': 1080, - }], - {}, + }] ), ( # https://github.com/ytdl-org/youtube-dl/pull/14844 'urls_only', @@ -1080,8 +973,7 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/ 'tbr': 4400, 'width': 1920, 'height': 1080, - }], - {}, + }] ), ( # https://github.com/ytdl-org/youtube-dl/issues/20346 # Media considered unfragmented even though it contains @@ -1127,185 +1019,18 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/ 'width': 360, 'height': 360, 'fps': 30, - }], - {}, - ), ( - # https://github.com/ytdl-org/youtube-dl/issues/30235 - # Bento4 generated test mpd - # mp4dash --mpd-name=manifest.mpd --no-split --use-segment-list mediafiles - 'url_and_range', - 'http://unknown/manifest.mpd', # mpd_url - 'http://unknown/', # mpd_base_url - [{ - 'manifest_url': 'http://unknown/manifest.mpd', - 'fragment_base_url': 'http://unknown/', - 'ext': 'm4a', - 'format_id': 'audio-und-mp4a.40.2', - 'format_note': 'DASH audio', - 'container': 'm4a_dash', - 'protocol': 'http_dash_segments', - 'acodec': 'mp4a.40.2', - 'vcodec': 'none', - 'tbr': 98.808, - }, { - 'manifest_url': 'http://unknown/manifest.mpd', - 'fragment_base_url': 'http://unknown/', - 'ext': 'mp4', - 'format_id': 'video-avc1', - 'format_note': 'DASH video', - 'container': 'mp4_dash', - 'protocol': 'http_dash_segments', - 'acodec': 'none', - 'vcodec': 'avc1.4D401E', - 'tbr': 699.597, - 'width': 768, - 'height': 432 - }], - {}, - ), ( - # https://github.com/ytdl-org/youtube-dl/issues/27575 - # GPAC generated test mpd - # MP4Box -dash 10000 -single-file -out manifest.mpd mediafiles - 'range_only', - 'http://unknown/manifest.mpd', # mpd_url - 'http://unknown/', # mpd_base_url - [{ - 'manifest_url': 'http://unknown/manifest.mpd', - 'fragment_base_url': 'http://unknown/audio_dashinit.mp4', - 'ext': 'm4a', - 'format_id': '2', - 'format_note': 'DASH audio', - 'container': 'm4a_dash', - 'protocol': 'http_dash_segments', - 'acodec': 'mp4a.40.2', - 'vcodec': 'none', - 'tbr': 98.096, - }, { - 'manifest_url': 'http://unknown/manifest.mpd', - 'fragment_base_url': 'http://unknown/video_dashinit.mp4', - 'ext': 'mp4', - 'format_id': '1', - 'format_note': 'DASH video', - 'container': 'mp4_dash', - 'protocol': 'http_dash_segments', - 'acodec': 'none', - 'vcodec': 'avc1.4D401E', - 'tbr': 526.987, - 'width': 768, - 'height': 432 - }], - {}, - ), ( - 'subtitles', - 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd', - 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/', - [{ - 'format_id': 'audio=128001', - 'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd', - 'ext': 'm4a', - 'tbr': 128.001, - 'asr': 48000, - 'format_note': 'DASH audio', - 'container': 'm4a_dash', - 'vcodec': 'none', - 'acodec': 'mp4a.40.2', - 'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd', - 'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/', - 'protocol': 'http_dash_segments', - }, { - 'format_id': 'video=100000', - 'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd', - 'ext': 'mp4', - 'width': 336, - 'height': 144, - 'tbr': 100, - 'format_note': 'DASH video', - 'container': 'mp4_dash', - 'vcodec': 'avc1.4D401F', - 'acodec': 'none', - 'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd', - 'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/', - 'protocol': 'http_dash_segments', - }, { - 'format_id': 'video=326000', - 'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd', - 'ext': 'mp4', - 'width': 562, - 'height': 240, - 'tbr': 326, - 'format_note': 'DASH video', - 'container': 'mp4_dash', - 'vcodec': 'avc1.4D401F', - 'acodec': 'none', - 'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd', - 'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/', - 'protocol': 'http_dash_segments', - }, { - 'format_id': 'video=698000', - 'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd', - 'ext': 'mp4', - 'width': 844, - 'height': 360, - 'tbr': 698, - 'format_note': 'DASH video', - 'container': 'mp4_dash', - 'vcodec': 'avc1.4D401F', - 'acodec': 'none', - 'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd', - 'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/', - 'protocol': 'http_dash_segments', - }, { - 'format_id': 'video=1493000', - 'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd', - 'ext': 'mp4', - 'width': 1126, - 'height': 480, - 'tbr': 1493, - 'format_note': 'DASH video', - 'container': 'mp4_dash', - 'vcodec': 'avc1.4D401F', - 'acodec': 'none', - 'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd', - 'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/', - 'protocol': 'http_dash_segments', - }, { - 'format_id': 'video=4482000', - 'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd', - 'ext': 'mp4', - 'width': 1688, - 'height': 720, - 'tbr': 4482, - 'format_note': 'DASH video', - 'container': 'mp4_dash', - 'vcodec': 'avc1.4D401F', - 'acodec': 'none', - 'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd', - 'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/', - 'protocol': 'http_dash_segments', - }], - { - 'en': [ - { - 'ext': 'mp4', - 'manifest_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd', - 'url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/manifest.mpd', - 'fragment_base_url': 'https://sdn-global-streaming-cache-3qsdn.akamaized.net/stream/3144/files/17/07/672975/3144-kZT4LWMQw6Rh7Kpd.ism/dash/', - 'protocol': 'http_dash_segments', - } - ] - }, + }] ) ] - for mpd_file, mpd_url, mpd_base_url, expected_formats, expected_subtitles in _TEST_CASES: - with open('./test/testdata/mpd/%s.mpd' % mpd_file, - mode='r', encoding='utf-8') as f: - formats, subtitles = self.ie._parse_mpd_formats_and_subtitles( + for mpd_file, mpd_url, mpd_base_url, expected_formats in _TEST_CASES: + with io.open('./test/testdata/mpd/%s.mpd' % mpd_file, + mode='r', encoding='utf-8') as f: + formats = self.ie._parse_mpd_formats( compat_etree_fromstring(f.read().encode('utf-8')), mpd_base_url=mpd_base_url, mpd_url=mpd_url) self.ie._sort_formats(formats) expect_value(self, formats, expected_formats, None) - expect_value(self, subtitles, expected_subtitles, None) def test_parse_f4m_formats(self): _TEST_CASES = [ @@ -1326,8 +1051,8 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/ ] for f4m_file, f4m_url, expected_formats in _TEST_CASES: - with open('./test/testdata/f4m/%s.f4m' % f4m_file, - mode='r', encoding='utf-8') as f: + with io.open('./test/testdata/f4m/%s.f4m' % f4m_file, + mode='r', encoding='utf-8') as f: formats = self.ie._parse_f4m_formats( compat_etree_fromstring(f.read().encode('utf-8')), f4m_url, None) @@ -1374,8 +1099,8 @@ jwplayer("mediaplayer").setup({"abouttext":"Visit Indie DB","aboutlink":"http:\/ ] for xspf_file, xspf_url, expected_entries in _TEST_CASES: - with open('./test/testdata/xspf/%s.xspf' % xspf_file, - mode='r', encoding='utf-8') as f: + with io.open('./test/testdata/xspf/%s.xspf' % xspf_file, + mode='r', encoding='utf-8') as f: entries = self.ie._parse_xspf( compat_etree_fromstring(f.read().encode('utf-8')), xspf_file, xspf_url=xspf_url, xspf_base_url=xspf_url) diff --git a/test/test_YoutubeDL.py b/test/test_YoutubeDL.py index d994682b2..a35effe0e 100644 --- a/test/test_YoutubeDL.py +++ b/test/test_YoutubeDL.py @@ -10,31 +10,14 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) import copy -import json -from test.helper import ( - FakeYDL, - assertRegexpMatches, - try_rm, -) +from test.helper import FakeYDL, assertRegexpMatches from youtube_dl import YoutubeDL -from youtube_dl.compat import ( - compat_http_cookiejar_Cookie, - compat_http_cookies_SimpleCookie, - compat_kwargs, - compat_open as open, - compat_str, - compat_urllib_error, -) - +from youtube_dl.compat import compat_str, compat_urllib_error from youtube_dl.extractor import YoutubeIE from youtube_dl.extractor.common import InfoExtractor from youtube_dl.postprocessor.common import PostProcessor -from youtube_dl.utils import ( - ExtractorError, - match_filter_func, - traverse_obj, -) +from youtube_dl.utils import ExtractorError, match_filter_func TEST_URL = 'http://localhost/sample.mp4' @@ -46,14 +29,11 @@ class YDL(FakeYDL): self.msgs = [] def process_info(self, info_dict): - self.downloaded_info_dicts.append(info_dict.copy()) + self.downloaded_info_dicts.append(info_dict) def to_screen(self, msg): self.msgs.append(msg) - def dl(self, *args, **kwargs): - assert False, 'Downloader must not be invoked for test_YoutubeDL' - def _make_result(formats, **kwargs): res = { @@ -62,9 +42,8 @@ def _make_result(formats, **kwargs): 'title': 'testttitle', 'extractor': 'testex', 'extractor_key': 'TestEx', - 'webpage_url': 'http://example.com/watch?v=shenanigans', } - res.update(**compat_kwargs(kwargs)) + res.update(**kwargs) return res @@ -702,12 +681,12 @@ class TestYoutubeDL(unittest.TestCase): class SimplePP(PostProcessor): def run(self, info): - with open(audiofile, 'w') as f: + with open(audiofile, 'wt') as f: f.write('EXAMPLE') return [info['filepath']], info def run_pp(params, PP): - with open(filename, 'w') as f: + with open(filename, 'wt') as f: f.write('EXAMPLE') ydl = YoutubeDL(params) ydl.add_post_processor(PP()) @@ -726,7 +705,7 @@ class TestYoutubeDL(unittest.TestCase): class ModifierPP(PostProcessor): def run(self, info): - with open(info['filepath'], 'w') as f: + with open(info['filepath'], 'wt') as f: f.write('MODIFIED') return [], info @@ -951,11 +930,17 @@ class TestYoutubeDL(unittest.TestCase): # Test case for https://github.com/ytdl-org/youtube-dl/issues/27064 def test_ignoreerrors_for_playlist_with_url_transparent_iterable_entries(self): - ydl = YDL({ + class _YDL(YDL): + def __init__(self, *args, **kwargs): + super(_YDL, self).__init__(*args, **kwargs) + + def trouble(self, s, tb=None): + pass + + ydl = _YDL({ 'format': 'extra', 'ignoreerrors': True, }) - ydl.trouble = lambda *_, **__: None class VideoIE(InfoExtractor): _VALID_URL = r'video:(?P\d+)' @@ -1012,180 +997,6 @@ class TestYoutubeDL(unittest.TestCase): self.assertEqual(downloaded['extractor'], 'Video') self.assertEqual(downloaded['extractor_key'], 'Video') - def test_default_times(self): - """Test addition of missing upload/release/_date from /release_/timestamp""" - info = { - 'id': '1234', - 'url': TEST_URL, - 'title': 'Title', - 'ext': 'mp4', - 'timestamp': 1631352900, - 'release_timestamp': 1632995931, - } - - params = {'simulate': True, } - ydl = FakeYDL(params) - out_info = ydl.process_ie_result(info) - self.assertTrue(isinstance(out_info['upload_date'], compat_str)) - self.assertEqual(out_info['upload_date'], '20210911') - self.assertTrue(isinstance(out_info['release_date'], compat_str)) - self.assertEqual(out_info['release_date'], '20210930') - - -class TestYoutubeDLCookies(unittest.TestCase): - - @staticmethod - def encode_cookie(cookie): - if not isinstance(cookie, dict): - cookie = vars(cookie) - for name, value in cookie.items(): - yield name, compat_str(value) - - @classmethod - def comparable_cookies(cls, cookies): - # Work around cookiejar cookies not being unicode strings - return sorted(map(tuple, map(sorted, map(cls.encode_cookie, cookies)))) - - def assertSameCookies(self, c1, c2, msg=None): - return self.assertEqual( - *map(self.comparable_cookies, (c1, c2)), - msg=msg) - - def assertSameCookieStrings(self, c1, c2, msg=None): - return self.assertSameCookies( - *map(lambda c: compat_http_cookies_SimpleCookie(c).values(), (c1, c2)), - msg=msg) - - def test_header_cookies(self): - - ydl = FakeYDL() - ydl.report_warning = lambda *_, **__: None - - def cookie(name, value, version=None, domain='', path='', secure=False, expires=None): - return compat_http_cookiejar_Cookie( - version or 0, name, value, None, False, - domain, bool(domain), bool(domain), path, bool(path), - secure, expires, False, None, None, rest={}) - - test_url, test_domain = (t % ('yt.dl',) for t in ('https://%s/test', '.%s')) - - def test(encoded_cookies, cookies, headers=False, round_trip=None, error_re=None): - def _test(): - ydl.cookiejar.clear() - ydl._load_cookies(encoded_cookies, autoscope=headers) - if headers: - ydl._apply_header_cookies(test_url) - data = {'url': test_url} - ydl._calc_headers(data) - self.assertSameCookies( - cookies, ydl.cookiejar, - 'Extracted cookiejar.Cookie is not the same') - if not headers: - self.assertSameCookieStrings( - data.get('cookies'), round_trip or encoded_cookies, - msg='Cookie is not the same as round trip') - ydl.__dict__['_YoutubeDL__header_cookies'] = [] - - try: - _test() - except AssertionError: - raise - except Exception as e: - if not error_re: - raise - assertRegexpMatches(self, e.args[0], error_re.join(('.*',) * 2)) - - test('test=value; Domain=' + test_domain, [cookie('test', 'value', domain=test_domain)]) - test('test=value', [cookie('test', 'value')], error_re='Unscoped cookies are not allowed') - test('cookie1=value1; Domain={0}; Path=/test; cookie2=value2; Domain={0}; Path=/'.format(test_domain), [ - cookie('cookie1', 'value1', domain=test_domain, path='/test'), - cookie('cookie2', 'value2', domain=test_domain, path='/')]) - cookie_kw = compat_kwargs( - {'domain': test_domain, 'path': '/test', 'secure': True, 'expires': '9999999999', }) - test('test=value; Domain={domain}; Path={path}; Secure; Expires={expires}'.format(**cookie_kw), [ - cookie('test', 'value', **cookie_kw)]) - test('test="value; "; path=/test; domain=' + test_domain, [ - cookie('test', 'value; ', domain=test_domain, path='/test')], - round_trip='test="value\\073 "; Domain={0}; Path=/test'.format(test_domain)) - test('name=; Domain=' + test_domain, [cookie('name', '', domain=test_domain)], - round_trip='name=""; Domain=' + test_domain) - test('test=value', [cookie('test', 'value', domain=test_domain)], headers=True) - test('cookie1=value; Domain={0}; cookie2=value'.format(test_domain), [], - headers=True, error_re='Invalid syntax') - ydl.report_warning = ydl.report_error - test('test=value', [], headers=True, error_re='Passing cookies as a header is a potential security risk') - - def test_infojson_cookies(self): - TEST_FILE = 'test_infojson_cookies.info.json' - TEST_URL = 'https://example.com/example.mp4' - COOKIES = 'a=b; Domain=.example.com; c=d; Domain=.example.com' - COOKIE_HEADER = {'Cookie': 'a=b; c=d'} - - ydl = FakeYDL() - ydl.process_info = lambda x: ydl._write_info_json('test', x, TEST_FILE) - - def make_info(info_header_cookies=False, fmts_header_cookies=False, cookies_field=False): - fmt = {'url': TEST_URL} - if fmts_header_cookies: - fmt['http_headers'] = COOKIE_HEADER - if cookies_field: - fmt['cookies'] = COOKIES - return _make_result([fmt], http_headers=COOKIE_HEADER if info_header_cookies else None) - - def test(initial_info, note): - - def failure_msg(why): - return ' when '.join((why, note)) - - result = {} - result['processed'] = ydl.process_ie_result(initial_info) - self.assertTrue(ydl.cookiejar.get_cookies_for_url(TEST_URL), - msg=failure_msg('No cookies set in cookiejar after initial process')) - ydl.cookiejar.clear() - with open(TEST_FILE) as infojson: - result['loaded'] = ydl.sanitize_info(json.load(infojson), True) - result['final'] = ydl.process_ie_result(result['loaded'].copy(), download=False) - self.assertTrue(ydl.cookiejar.get_cookies_for_url(TEST_URL), - msg=failure_msg('No cookies set in cookiejar after final process')) - ydl.cookiejar.clear() - for key in ('processed', 'loaded', 'final'): - info = result[key] - self.assertIsNone( - traverse_obj(info, ((None, ('formats', 0)), 'http_headers', 'Cookie'), casesense=False, get_all=False), - msg=failure_msg('Cookie header not removed in {0} result'.format(key))) - self.assertSameCookieStrings( - traverse_obj(info, ((None, ('formats', 0)), 'cookies'), get_all=False), COOKIES, - msg=failure_msg('No cookies field found in {0} result'.format(key))) - - test({'url': TEST_URL, 'http_headers': COOKIE_HEADER, 'id': '1', 'title': 'x'}, 'no formats field') - test(make_info(info_header_cookies=True), 'info_dict header cokies') - test(make_info(fmts_header_cookies=True), 'format header cookies') - test(make_info(info_header_cookies=True, fmts_header_cookies=True), 'info_dict and format header cookies') - test(make_info(info_header_cookies=True, fmts_header_cookies=True, cookies_field=True), 'all cookies fields') - test(make_info(cookies_field=True), 'cookies format field') - test({'url': TEST_URL, 'cookies': COOKIES, 'id': '1', 'title': 'x'}, 'info_dict cookies field only') - - try_rm(TEST_FILE) - - def test_add_headers_cookie(self): - def check_for_cookie_header(result): - return traverse_obj(result, ((None, ('formats', 0)), 'http_headers', 'Cookie'), casesense=False, get_all=False) - - ydl = FakeYDL({'http_headers': {'Cookie': 'a=b'}}) - ydl._apply_header_cookies(_make_result([])['webpage_url']) # Scope to input webpage URL: .example.com - - fmt = {'url': 'https://example.com/video.mp4'} - result = ydl.process_ie_result(_make_result([fmt]), download=False) - self.assertIsNone(check_for_cookie_header(result), msg='http_headers cookies in result info_dict') - self.assertEqual(result.get('cookies'), 'a=b; Domain=.example.com', msg='No cookies were set in cookies field') - self.assertIn('a=b', ydl.cookiejar.get_cookie_header(fmt['url']), msg='No cookies were set in cookiejar') - - fmt = {'url': 'https://wrong.com/video.mp4'} - result = ydl.process_ie_result(_make_result([fmt]), download=False) - self.assertIsNone(check_for_cookie_header(result), msg='http_headers cookies for wrong domain') - self.assertFalse(result.get('cookies'), msg='Cookies set in cookies field for wrong domain') - self.assertFalse(ydl.cookiejar.get_cookie_header(fmt['url']), msg='Cookies set in cookiejar for wrong domain') - if __name__ == '__main__': unittest.main() diff --git a/test/test_YoutubeDLCookieJar.py b/test/test_YoutubeDLCookieJar.py index 4f9dd71ae..05f48bd74 100644 --- a/test/test_YoutubeDLCookieJar.py +++ b/test/test_YoutubeDLCookieJar.py @@ -46,20 +46,6 @@ class TestYoutubeDLCookieJar(unittest.TestCase): # will be ignored self.assertFalse(cookiejar._cookies) - def test_get_cookie_header(self): - cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/httponly_cookies.txt') - cookiejar.load(ignore_discard=True, ignore_expires=True) - header = cookiejar.get_cookie_header('https://www.foobar.foobar') - self.assertIn('HTTPONLY_COOKIE', header) - - def test_get_cookies_for_url(self): - cookiejar = YoutubeDLCookieJar('./test/testdata/cookies/session_cookies.txt') - cookiejar.load(ignore_discard=True, ignore_expires=True) - cookies = cookiejar.get_cookies_for_url('https://www.foobar.foobar/') - self.assertEqual(len(cookies), 2) - cookies = cookiejar.get_cookies_for_url('https://foobar.foobar/') - self.assertFalse(cookies) - if __name__ == '__main__': unittest.main() diff --git a/test/test_aes.py b/test/test_aes.py index 0f181466b..cc89fb6ab 100644 --- a/test/test_aes.py +++ b/test/test_aes.py @@ -8,7 +8,7 @@ import sys import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from youtube_dl.aes import aes_decrypt, aes_encrypt, aes_cbc_decrypt, aes_cbc_encrypt, aes_decrypt_text, aes_ecb_encrypt +from youtube_dl.aes import aes_decrypt, aes_encrypt, aes_cbc_decrypt, aes_cbc_encrypt, aes_decrypt_text from youtube_dl.utils import bytes_to_intlist, intlist_to_bytes import base64 @@ -58,13 +58,6 @@ class TestAES(unittest.TestCase): decrypted = (aes_decrypt_text(encrypted, password, 32)) self.assertEqual(decrypted, self.secret_msg) - def test_ecb_encrypt(self): - data = bytes_to_intlist(self.secret_msg) - encrypted = intlist_to_bytes(aes_ecb_encrypt(data, self.key)) - self.assertEqual( - encrypted, - b'\xaa\x86]\x81\x97>\x02\x92\x9d\x1bR[[L/u\xd3&\xd1(h\xde{\x81\x94\xba\x02\xae\xbd\xa6\xd0:') - if __name__ == '__main__': unittest.main() diff --git a/test/test_age_restriction.py b/test/test_age_restriction.py index db98494ab..6f5513faa 100644 --- a/test/test_age_restriction.py +++ b/test/test_age_restriction.py @@ -11,7 +11,6 @@ from test.helper import try_rm from youtube_dl import YoutubeDL -from youtube_dl.utils import DownloadError def _download_restricted(url, filename, age): @@ -27,10 +26,7 @@ def _download_restricted(url, filename, age): ydl.add_default_info_extractors() json_filename = os.path.splitext(filename)[0] + '.info.json' try_rm(json_filename) - try: - ydl.download([url]) - except DownloadError: - try_rm(json_filename) + ydl.download([url]) res = os.path.exists(json_filename) try_rm(json_filename) return res @@ -42,12 +38,12 @@ class TestAgeRestriction(unittest.TestCase): self.assertFalse(_download_restricted(url, filename, age)) def test_youtube(self): - self._assert_restricted('HtVdAasjOgU', 'HtVdAasjOgU.mp4', 10) + self._assert_restricted('07FYdnEawAQ', '07FYdnEawAQ.mp4', 10) def test_youporn(self): self._assert_restricted( - 'https://www.youporn.com/watch/16715086/sex-ed-in-detention-18-asmr/', - '16715086.mp4', 2, old_age=25) + 'http://www.youporn.com/watch/505835/sex-ed-is-it-safe-to-masturbate-daily/', + '505835.mp4', 2, old_age=25) if __name__ == '__main__': diff --git a/test/test_all_urls.py b/test/test_all_urls.py index 26df356b4..df6d81b5d 100644 --- a/test/test_all_urls.py +++ b/test/test_all_urls.py @@ -66,9 +66,18 @@ class TestAllURLsMatching(unittest.TestCase): self.assertMatch('https://www.youtube.com/feed/watch_later', ['youtube:tab']) self.assertMatch('https://www.youtube.com/feed/subscriptions', ['youtube:tab']) - def test_youtube_search_matching(self): - self.assertMatch('http://www.youtube.com/results?search_query=making+mustard', ['youtube:search_url']) - self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url']) + # def test_youtube_search_matching(self): + # self.assertMatch('http://www.youtube.com/results?search_query=making+mustard', ['youtube:search_url']) + # self.assertMatch('https://www.youtube.com/results?baz=bar&search_query=youtube-dl+test+video&filters=video&lclk=video', ['youtube:search_url']) + + def test_youtube_extract(self): + assertExtractId = lambda url, id: self.assertEqual(YoutubeIE.extract_id(url), id) + assertExtractId('http://www.youtube.com/watch?&v=BaW_jenozKc', 'BaW_jenozKc') + assertExtractId('https://www.youtube.com/watch?&v=BaW_jenozKc', 'BaW_jenozKc') + assertExtractId('https://www.youtube.com/watch?feature=player_embedded&v=BaW_jenozKc', 'BaW_jenozKc') + assertExtractId('https://www.youtube.com/watch_popup?v=BaW_jenozKc', 'BaW_jenozKc') + assertExtractId('http://www.youtube.com/watch?v=BaW_jenozKcsharePLED17F32AD9753930', 'BaW_jenozKc') + assertExtractId('BaW_jenozKc', 'BaW_jenozKc') def test_facebook_matching(self): self.assertTrue(FacebookIE.suitable('https://www.facebook.com/Shiniknoh#!/photo.php?v=10153317450565268')) diff --git a/test/test_cache.py b/test/test_cache.py index 0431f4f15..a16160142 100644 --- a/test/test_cache.py +++ b/test/test_cache.py @@ -3,18 +3,17 @@ from __future__ import unicode_literals +import shutil + # Allow direct execution import os import sys import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import shutil from test.helper import FakeYDL from youtube_dl.cache import Cache -from youtube_dl.utils import version_tuple -from youtube_dl.version import __version__ def _is_empty(d): @@ -55,29 +54,6 @@ class TestCache(unittest.TestCase): self.assertFalse(os.path.exists(self.test_dir)) self.assertEqual(c.load('test_cache', 'k.'), None) - def test_cache_validation(self): - ydl = FakeYDL({ - 'cachedir': self.test_dir, - }) - c = Cache(ydl) - obj = {'x': 1, 'y': ['ä', '\\a', True]} - c.store('test_cache', 'k.', obj) - self.assertEqual(c.load('test_cache', 'k.', min_ver='1970.01.01'), obj) - new_version = '.'.join(('%0.2d' % ((v + 1) if i == 0 else v, )) for i, v in enumerate(version_tuple(__version__))) - self.assertIs(c.load('test_cache', 'k.', min_ver=new_version), None) - - def test_cache_clear(self): - ydl = FakeYDL({ - 'cachedir': self.test_dir, - }) - c = Cache(ydl) - c.store('test_cache', 'k.', 'kay') - c.store('test_cache', 'l.', 'ell') - self.assertEqual(c.load('test_cache', 'k.'), 'kay') - c.clear('test_cache', 'k.') - self.assertEqual(c.load('test_cache', 'k.'), None) - self.assertEqual(c.load('test_cache', 'l.'), 'ell') - if __name__ == '__main__': unittest.main() diff --git a/test/test_compat.py b/test/test_compat.py index b83c8cb41..86ff389fd 100644 --- a/test/test_compat.py +++ b/test/test_compat.py @@ -11,7 +11,6 @@ sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from youtube_dl.compat import ( - compat_casefold, compat_getenv, compat_setenv, compat_etree_Element, @@ -23,7 +22,6 @@ from youtube_dl.compat import ( compat_urllib_parse_unquote, compat_urllib_parse_unquote_plus, compat_urllib_parse_urlencode, - compat_urllib_request, ) @@ -49,11 +47,10 @@ class TestCompat(unittest.TestCase): def test_all_present(self): import youtube_dl.compat - all_names = sorted( - youtube_dl.compat.__all__ + youtube_dl.compat.legacy) - present_names = set(map(compat_str, filter( + all_names = youtube_dl.compat.__all__ + present_names = set(filter( lambda c: '_' in c and not c.startswith('_'), - dir(youtube_dl.compat)))) - set(['unicode_literals']) + dir(youtube_dl.compat))) - set(['unicode_literals']) self.assertEqual(all_names, sorted(present_names)) def test_compat_urllib_parse_unquote(self): @@ -121,34 +118,9 @@ class TestCompat(unittest.TestCase): ''' compat_etree_fromstring(xml) - def test_compat_struct_unpack(self): + def test_struct_unpack(self): self.assertEqual(compat_struct_unpack('!B', b'\x00'), (0,)) - def test_compat_casefold(self): - if hasattr(compat_str, 'casefold'): - # don't bother to test str.casefold() (again) - return - # thanks https://bugs.python.org/file24232/casefolding.patch - self.assertEqual(compat_casefold('hello'), 'hello') - self.assertEqual(compat_casefold('hELlo'), 'hello') - self.assertEqual(compat_casefold('ß'), 'ss') - self.assertEqual(compat_casefold('fi'), 'fi') - self.assertEqual(compat_casefold('\u03a3'), '\u03c3') - self.assertEqual(compat_casefold('A\u0345\u03a3'), 'a\u03b9\u03c3') - - def test_compat_urllib_request_Request(self): - self.assertEqual( - compat_urllib_request.Request('http://127.0.0.1', method='PUT').get_method(), - 'PUT') - - class PUTrequest(compat_urllib_request.Request): - def get_method(self): - return 'PUT' - - self.assertEqual( - PUTrequest('http://127.0.0.1').get_method(), - 'PUT') - if __name__ == '__main__': unittest.main() diff --git a/test/test_download.py b/test/test_download.py index f7d6a23bc..ebe820dfc 100644 --- a/test/test_download.py +++ b/test/test_download.py @@ -9,6 +9,7 @@ import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from test.helper import ( + assertGreaterEqual, expect_warnings, get_params, gettestcases, @@ -19,35 +20,26 @@ from test.helper import ( import hashlib +import io import json import socket import youtube_dl.YoutubeDL from youtube_dl.compat import ( compat_http_client, - compat_HTTPError, - compat_open as open, compat_urllib_error, + compat_HTTPError, ) from youtube_dl.utils import ( DownloadError, ExtractorError, - error_to_compat_str, format_bytes, - IDENTITY, - preferredencoding, UnavailableVideoError, ) from youtube_dl.extractor import get_info_extractor RETRIES = 3 -# Some unittest APIs require actual str -if not isinstance('TEST', str): - _encode_str = lambda s: s.encode(preferredencoding()) -else: - _encode_str = IDENTITY - class YoutubeDL(youtube_dl.YoutubeDL): def __init__(self, *args, **kwargs): @@ -108,31 +100,27 @@ def generator(test_case, tname): def print_skipping(reason): print('Skipping %s: %s' % (test_case['name'], reason)) - self.skipTest(_encode_str(reason)) - if not ie.working(): print_skipping('IE marked as not _WORKING') + return for tc in test_cases: info_dict = tc.get('info_dict', {}) if not (info_dict.get('id') and info_dict.get('ext')): - raise Exception('Test definition (%s) requires both \'id\' and \'ext\' keys present to define the output file' % (tname, )) + raise Exception('Test definition incorrect. The output file cannot be known. Are both \'id\' and \'ext\' keys present?') if 'skip' in test_case: print_skipping(test_case['skip']) - + return for other_ie in other_ies: if not other_ie.working(): print_skipping('test depends on %sIE, marked as not WORKING' % other_ie.ie_key()) + return params = get_params(test_case.get('params', {})) params['outtmpl'] = tname + '_' + params['outtmpl'] if is_playlist and 'playlist' not in test_case: params.setdefault('extract_flat', 'in_playlist') - params.setdefault('playlistend', - test_case['playlist_maxcount'] + 1 - if test_case.get('playlist_maxcount') - else test_case.get('playlist_mincount')) params.setdefault('skip_download', True) ydl = YoutubeDL(params, auto_init=False) @@ -158,7 +146,6 @@ def generator(test_case, tname): try_rm(tc_filename) try_rm(tc_filename + '.part') try_rm(os.path.splitext(tc_filename)[0] + '.info.json') - try_rm_tcs_files() try: try_num = 1 @@ -173,9 +160,7 @@ def generator(test_case, tname): except (DownloadError, ExtractorError) as err: # Check if the exception is not a network related one if not err.exc_info[0] in (compat_urllib_error.URLError, socket.timeout, UnavailableVideoError, compat_http_client.BadStatusLine) or (err.exc_info[0] == compat_HTTPError and err.exc_info[1].code == 503): - msg = getattr(err, 'msg', error_to_compat_str(err)) - err.msg = '%s (%s)' % (msg, tname, ) - raise err + raise if try_num == RETRIES: report_warning('%s failed due to network errors, skipping...' % tname) @@ -193,19 +178,13 @@ def generator(test_case, tname): expect_info_dict(self, res_dict, test_case.get('info_dict', {})) if 'playlist_mincount' in test_case: - self.assertGreaterEqual( + assertGreaterEqual( + self, len(res_dict['entries']), test_case['playlist_mincount'], 'Expected at least %d in playlist %s, but got only %d' % ( test_case['playlist_mincount'], test_case['url'], len(res_dict['entries']))) - if 'playlist_maxcount' in test_case: - self.assertLessEqual( - len(res_dict['entries']), - test_case['playlist_maxcount'], - 'Expected at most %d in playlist %s, but got %d' % ( - test_case['playlist_maxcount'], test_case['url'], - len(res_dict['entries']))) if 'playlist_count' in test_case: self.assertEqual( len(res_dict['entries']), @@ -230,15 +209,7 @@ def generator(test_case, tname): # First, check test cases' data against extracted data alone expect_info_dict(self, tc_res_dict, tc.get('info_dict', {})) # Now, check downloaded file consistency - # support test-case with volatile ID, signalled by regexp value - if tc.get('info_dict', {}).get('id', '').startswith('re:'): - test_id = tc['info_dict']['id'] - tc['info_dict']['id'] = tc_res_dict['id'] - else: - test_id = None tc_filename = get_tc_filename(tc) - if test_id: - tc['info_dict']['id'] = test_id if not test_case.get('params', {}).get('skip_download', False): self.assertTrue(os.path.exists(tc_filename), msg='Missing file ' + tc_filename) self.assertTrue(tc_filename in finished_hook_called) @@ -247,8 +218,8 @@ def generator(test_case, tname): if params.get('test'): expected_minsize = max(expected_minsize, 10000) got_fsize = os.path.getsize(tc_filename) - self.assertGreaterEqual( - got_fsize, expected_minsize, + assertGreaterEqual( + self, got_fsize, expected_minsize, 'Expected %s to be at least %s, but it\'s only %s ' % (tc_filename, format_bytes(expected_minsize), format_bytes(got_fsize))) @@ -261,7 +232,7 @@ def generator(test_case, tname): self.assertTrue( os.path.exists(info_json_fn), 'Missing info file %s' % info_json_fn) - with open(info_json_fn, encoding='utf-8') as infof: + with io.open(info_json_fn, encoding='utf-8') as infof: info_dict = json.load(infof) expect_info_dict(self, info_dict, tc.get('info_dict', {})) finally: diff --git a/test/test_downloader_external.py b/test/test_downloader_external.py deleted file mode 100644 index 4491bd9de..000000000 --- a/test/test_downloader_external.py +++ /dev/null @@ -1,272 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 -from __future__ import unicode_literals - -# Allow direct execution -import os -import re -import sys -import subprocess -import unittest -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -from test.helper import ( - FakeLogger, - FakeYDL, - http_server_port, - try_rm, -) -from youtube_dl import YoutubeDL -from youtube_dl.compat import ( - compat_contextlib_suppress, - compat_http_cookiejar_Cookie, - compat_http_server, - compat_kwargs, -) -from youtube_dl.utils import ( - encodeFilename, - join_nonempty, -) -from youtube_dl.downloader.external import ( - Aria2cFD, - Aria2pFD, - AxelFD, - CurlFD, - FFmpegFD, - HttpieFD, - WgetFD, -) -from youtube_dl.postprocessor import ( - FFmpegPostProcessor, -) -import threading - -TEST_SIZE = 10 * 1024 - -TEST_COOKIE = { - 'version': 0, - 'name': 'test', - 'value': 'ytdlp', - 'port': None, - 'port_specified': False, - 'domain': '.example.com', - 'domain_specified': True, - 'domain_initial_dot': False, - 'path': '/', - 'path_specified': True, - 'secure': False, - 'expires': None, - 'discard': False, - 'comment': None, - 'comment_url': None, - 'rest': {}, -} - -TEST_COOKIE_VALUE = join_nonempty('name', 'value', delim='=', from_dict=TEST_COOKIE) - -TEST_INFO = {'url': 'http://www.example.com/'} - - -def cookiejar_Cookie(**cookie_args): - return compat_http_cookiejar_Cookie(**compat_kwargs(cookie_args)) - - -def ifExternalFDAvailable(externalFD): - return unittest.skipUnless(externalFD.available(), - externalFD.get_basename() + ' not found') - - -class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler): - def log_message(self, format, *args): - pass - - def send_content_range(self, total=None): - range_header = self.headers.get('Range') - start = end = None - if range_header: - mobj = re.match(r'bytes=(\d+)-(\d+)', range_header) - if mobj: - start, end = (int(mobj.group(i)) for i in (1, 2)) - valid_range = start is not None and end is not None - if valid_range: - content_range = 'bytes %d-%d' % (start, end) - if total: - content_range += '/%d' % total - self.send_header('Content-Range', content_range) - return (end - start + 1) if valid_range else total - - def serve(self, range=True, content_length=True): - self.send_response(200) - self.send_header('Content-Type', 'video/mp4') - size = TEST_SIZE - if range: - size = self.send_content_range(TEST_SIZE) - if content_length: - self.send_header('Content-Length', size) - self.end_headers() - self.wfile.write(b'#' * size) - - def do_GET(self): - if self.path == '/regular': - self.serve() - elif self.path == '/no-content-length': - self.serve(content_length=False) - elif self.path == '/no-range': - self.serve(range=False) - elif self.path == '/no-range-no-content-length': - self.serve(range=False, content_length=False) - else: - assert False, 'unrecognised server path' - - -@ifExternalFDAvailable(Aria2pFD) -class TestAria2pFD(unittest.TestCase): - def setUp(self): - self.httpd = compat_http_server.HTTPServer( - ('127.0.0.1', 0), HTTPTestRequestHandler) - self.port = http_server_port(self.httpd) - self.server_thread = threading.Thread(target=self.httpd.serve_forever) - self.server_thread.daemon = True - self.server_thread.start() - - def download(self, params, ep): - with subprocess.Popen( - ['aria2c', '--enable-rpc'], - stdout=subprocess.DEVNULL, - stderr=subprocess.DEVNULL - ) as process: - if not process.poll(): - filename = 'testfile.mp4' - params['logger'] = FakeLogger() - params['outtmpl'] = filename - ydl = YoutubeDL(params) - try_rm(encodeFilename(filename)) - self.assertEqual(ydl.download(['http://127.0.0.1:%d/%s' % (self.port, ep)]), 0) - self.assertEqual(os.path.getsize(encodeFilename(filename)), TEST_SIZE) - try_rm(encodeFilename(filename)) - process.kill() - - def download_all(self, params): - for ep in ('regular', 'no-content-length', 'no-range', 'no-range-no-content-length'): - self.download(params, ep) - - def test_regular(self): - self.download_all({'external_downloader': 'aria2p'}) - - def test_chunked(self): - self.download_all({ - 'external_downloader': 'aria2p', - 'http_chunk_size': 1000, - }) - - -@ifExternalFDAvailable(HttpieFD) -class TestHttpieFD(unittest.TestCase): - def test_make_cmd(self): - with FakeYDL() as ydl: - downloader = HttpieFD(ydl, {}) - self.assertEqual( - downloader._make_cmd('test', TEST_INFO), - ['http', '--download', '--output', 'test', 'http://www.example.com/']) - - # Test cookie header is added - ydl.cookiejar.set_cookie(cookiejar_Cookie(**TEST_COOKIE)) - self.assertEqual( - downloader._make_cmd('test', TEST_INFO), - ['http', '--download', '--output', 'test', - 'http://www.example.com/', 'Cookie:' + TEST_COOKIE_VALUE]) - - -@ifExternalFDAvailable(AxelFD) -class TestAxelFD(unittest.TestCase): - def test_make_cmd(self): - with FakeYDL() as ydl: - downloader = AxelFD(ydl, {}) - self.assertEqual( - downloader._make_cmd('test', TEST_INFO), - ['axel', '-o', 'test', '--', 'http://www.example.com/']) - - # Test cookie header is added - ydl.cookiejar.set_cookie(cookiejar_Cookie(**TEST_COOKIE)) - self.assertEqual( - downloader._make_cmd('test', TEST_INFO), - ['axel', '-o', 'test', '-H', 'Cookie: ' + TEST_COOKIE_VALUE, - '--max-redirect=0', '--', 'http://www.example.com/']) - - -@ifExternalFDAvailable(WgetFD) -class TestWgetFD(unittest.TestCase): - def test_make_cmd(self): - with FakeYDL() as ydl: - downloader = WgetFD(ydl, {}) - self.assertNotIn('--load-cookies', downloader._make_cmd('test', TEST_INFO)) - # Test cookiejar tempfile arg is added - ydl.cookiejar.set_cookie(cookiejar_Cookie(**TEST_COOKIE)) - self.assertIn('--load-cookies', downloader._make_cmd('test', TEST_INFO)) - - -@ifExternalFDAvailable(CurlFD) -class TestCurlFD(unittest.TestCase): - def test_make_cmd(self): - with FakeYDL() as ydl: - downloader = CurlFD(ydl, {}) - self.assertNotIn('--cookie', downloader._make_cmd('test', TEST_INFO)) - # Test cookie header is added - ydl.cookiejar.set_cookie(cookiejar_Cookie(**TEST_COOKIE)) - self.assertIn('--cookie', downloader._make_cmd('test', TEST_INFO)) - self.assertIn(TEST_COOKIE_VALUE, downloader._make_cmd('test', TEST_INFO)) - - -@ifExternalFDAvailable(Aria2cFD) -class TestAria2cFD(unittest.TestCase): - def test_make_cmd(self): - with FakeYDL() as ydl: - downloader = Aria2cFD(ydl, {}) - downloader._make_cmd('test', TEST_INFO) - self.assertFalse(hasattr(downloader, '_cookies_tempfile')) - - # Test cookiejar tempfile arg is added - ydl.cookiejar.set_cookie(cookiejar_Cookie(**TEST_COOKIE)) - cmd = downloader._make_cmd('test', TEST_INFO) - self.assertIn('--load-cookies=%s' % downloader._cookies_tempfile, cmd) - - -# Handle delegated availability -def ifFFmpegFDAvailable(externalFD): - # raise SkipTest, or set False! - avail = ifExternalFDAvailable(externalFD) and False - with compat_contextlib_suppress(Exception): - avail = FFmpegPostProcessor(downloader=None).available - return unittest.skipUnless( - avail, externalFD.get_basename() + ' not found') - - -@ifFFmpegFDAvailable(FFmpegFD) -class TestFFmpegFD(unittest.TestCase): - _args = [] - - def _test_cmd(self, args): - self._args = args - - def test_make_cmd(self): - with FakeYDL() as ydl: - downloader = FFmpegFD(ydl, {}) - downloader._debug_cmd = self._test_cmd - info_dict = TEST_INFO.copy() - info_dict['ext'] = 'mp4' - - downloader._call_downloader('test', info_dict) - self.assertEqual(self._args, [ - 'ffmpeg', '-y', '-i', 'http://www.example.com/', - '-c', 'copy', '-f', 'mp4', 'file:test']) - - # Test cookies arg is added - ydl.cookiejar.set_cookie(cookiejar_Cookie(**TEST_COOKIE)) - downloader._call_downloader('test', info_dict) - self.assertEqual(self._args, [ - 'ffmpeg', '-y', '-cookies', TEST_COOKIE_VALUE + '; path=/; domain=.example.com;\r\n', - '-i', 'http://www.example.com/', '-c', 'copy', '-f', 'mp4', 'file:test']) - - -if __name__ == '__main__': - unittest.main() diff --git a/test/test_downloader_http.py b/test/test_downloader_http.py index 6af86ae48..750472281 100644 --- a/test/test_downloader_http.py +++ b/test/test_downloader_http.py @@ -9,11 +9,7 @@ import sys import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -from test.helper import ( - FakeLogger, - http_server_port, - try_rm, -) +from test.helper import http_server_port, try_rm from youtube_dl import YoutubeDL from youtube_dl.compat import compat_http_server from youtube_dl.downloader.http import HttpFD @@ -70,6 +66,17 @@ class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler): assert False +class FakeLogger(object): + def debug(self, msg): + pass + + def warning(self, msg): + pass + + def error(self, msg): + pass + + class TestHttpFD(unittest.TestCase): def setUp(self): self.httpd = compat_http_server.HTTPServer( @@ -88,7 +95,7 @@ class TestHttpFD(unittest.TestCase): self.assertTrue(downloader.real_download(filename, { 'url': 'http://127.0.0.1:%d/%s' % (self.port, ep), })) - self.assertEqual(os.path.getsize(encodeFilename(filename)), TEST_SIZE, ep) + self.assertEqual(os.path.getsize(encodeFilename(filename)), TEST_SIZE) try_rm(encodeFilename(filename)) def download_all(self, params): diff --git a/test/test_execution.py b/test/test_execution.py index 9daaafa6c..11661bb68 100644 --- a/test/test_execution.py +++ b/test/test_execution.py @@ -8,55 +8,37 @@ import unittest import sys import os import subprocess +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) + +from youtube_dl.utils import encodeArgument rootDir = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) -sys.path.insert(0, rootDir) -from youtube_dl.compat import compat_register_utf8, compat_subprocess_get_DEVNULL -from youtube_dl.utils import encodeArgument - -compat_register_utf8() - - -_DEV_NULL = compat_subprocess_get_DEVNULL() +try: + _DEV_NULL = subprocess.DEVNULL +except AttributeError: + _DEV_NULL = open(os.devnull, 'wb') class TestExecution(unittest.TestCase): - def setUp(self): - self.module = 'youtube_dl' - if sys.version_info < (2, 7): - self.module += '.__main__' - def test_import(self): subprocess.check_call([sys.executable, '-c', 'import youtube_dl'], cwd=rootDir) def test_module_exec(self): - subprocess.check_call([sys.executable, '-m', self.module, '--version'], cwd=rootDir, stdout=_DEV_NULL) + if sys.version_info >= (2, 7): # Python 2.6 doesn't support package execution + subprocess.check_call([sys.executable, '-m', 'youtube_dl', '--version'], cwd=rootDir, stdout=_DEV_NULL) def test_main_exec(self): - subprocess.check_call([sys.executable, os.path.normpath('youtube_dl/__main__.py'), '--version'], cwd=rootDir, stdout=_DEV_NULL) + subprocess.check_call([sys.executable, 'youtube_dl/__main__.py', '--version'], cwd=rootDir, stdout=_DEV_NULL) def test_cmdline_umlauts(self): - os.environ['PYTHONIOENCODING'] = 'utf-8' p = subprocess.Popen( - [sys.executable, '-m', self.module, encodeArgument('ä'), '--version'], + [sys.executable, 'youtube_dl/__main__.py', encodeArgument('ä'), '--version'], cwd=rootDir, stdout=_DEV_NULL, stderr=subprocess.PIPE) _, stderr = p.communicate() self.assertFalse(stderr) - def test_lazy_extractors(self): - lazy_extractors = os.path.normpath('youtube_dl/extractor/lazy_extractors.py') - try: - subprocess.check_call([sys.executable, os.path.normpath('devscripts/make_lazy_extractors.py'), lazy_extractors], cwd=rootDir, stdout=_DEV_NULL) - subprocess.check_call([sys.executable, os.path.normpath('test/test_all_urls.py')], cwd=rootDir, stdout=_DEV_NULL) - finally: - for x in ('', 'c') if sys.version_info[0] < 3 else ('',): - try: - os.remove(lazy_extractors + x) - except OSError: - pass - if __name__ == '__main__': unittest.main() diff --git a/test/test_http.py b/test/test_http.py index 485c4c6fc..3ee0a5dda 100644 --- a/test/test_http.py +++ b/test/test_http.py @@ -8,163 +8,30 @@ import sys import unittest sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import contextlib -import gzip -import io -import ssl -import tempfile -import threading -import zlib - -# avoid deprecated alias assertRaisesRegexp -if hasattr(unittest.TestCase, 'assertRaisesRegex'): - unittest.TestCase.assertRaisesRegexp = unittest.TestCase.assertRaisesRegex - -try: - import brotli -except ImportError: - brotli = None -try: - from urllib.request import pathname2url -except ImportError: - from urllib import pathname2url - -from youtube_dl.compat import ( - compat_http_cookiejar_Cookie, - compat_http_server, - compat_str as str, - compat_urllib_error, - compat_urllib_HTTPError, - compat_urllib_parse, - compat_urllib_request, -) - -from youtube_dl.utils import ( - sanitized_Request, - update_Request, - urlencode_postdata, -) - -from test.helper import ( - expectedFailureIf, - FakeYDL, - FakeLogger, - http_server_port, -) +from test.helper import http_server_port from youtube_dl import YoutubeDL +from youtube_dl.compat import compat_http_server, compat_urllib_request +import ssl +import threading TEST_DIR = os.path.dirname(os.path.abspath(__file__)) class HTTPTestRequestHandler(compat_http_server.BaseHTTPRequestHandler): - protocol_version = 'HTTP/1.1' - - # work-around old/new -style class inheritance - def super(self, meth_name, *args, **kwargs): - from types import MethodType - try: - super() - fn = lambda s, m, *a, **k: getattr(super(), m)(*a, **k) - except TypeError: - fn = lambda s, m, *a, **k: getattr(compat_http_server.BaseHTTPRequestHandler, m)(s, *a, **k) - self.super = MethodType(fn, self) - return self.super(meth_name, *args, **kwargs) - def log_message(self, format, *args): pass - def _headers(self): - payload = str(self.headers).encode('utf-8') - self.send_response(200) - self.send_header('Content-Type', 'application/json') - self.send_header('Content-Length', str(len(payload))) - self.end_headers() - self.wfile.write(payload) - - def _redirect(self): - self.send_response(int(self.path[len('/redirect_'):])) - self.send_header('Location', '/method') - self.send_header('Content-Length', '0') - self.end_headers() - - def _method(self, method, payload=None): - self.send_response(200) - self.send_header('Content-Length', str(len(payload or ''))) - self.send_header('Method', method) - self.end_headers() - if payload: - self.wfile.write(payload) - - def _status(self, status): - payload = '{0} NOT FOUND'.format(status).encode('utf-8') - self.send_response(int(status)) - self.send_header('Content-Type', 'text/html; charset=utf-8') - self.send_header('Content-Length', str(len(payload))) - self.end_headers() - self.wfile.write(payload) - - def _read_data(self): - if 'Content-Length' in self.headers: - return self.rfile.read(int(self.headers['Content-Length'])) - - def _test_url(self, path, host='127.0.0.1', scheme='http', port=None): - return '{0}://{1}:{2}/{3}'.format( - scheme, host, - port if port is not None - else http_server_port(self.server), path) - - def do_POST(self): - data = self._read_data() - if self.path.startswith('/redirect_'): - self._redirect() - elif self.path.startswith('/method'): - self._method('POST', data) - elif self.path.startswith('/headers'): - self._headers() - else: - self._status(404) - - def do_HEAD(self): - if self.path.startswith('/redirect_'): - self._redirect() - elif self.path.startswith('/method'): - self._method('HEAD') - else: - self._status(404) - - def do_PUT(self): - data = self._read_data() - if self.path.startswith('/redirect_'): - self._redirect() - elif self.path.startswith('/method'): - self._method('PUT', data) - else: - self._status(404) - def do_GET(self): - - def respond(payload=b'