diff --git a/.github/ISSUE_TEMPLATE/1_broken_site.md b/.github/ISSUE_TEMPLATE/1_broken_site.md index e5405c235..ce0319fe2 100644 --- a/.github/ISSUE_TEMPLATE/1_broken_site.md +++ b/.github/ISSUE_TEMPLATE/1_broken_site.md @@ -18,7 +18,7 @@ title: '' - [ ] I'm reporting a broken site support -- [ ] I've verified that I'm running youtube-dl version **2021.12.17** +- [ ] I've verified that I'm running youtube-dl version **2020.09.20** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar issues including closed ones @@ -41,7 +41,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v < [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version 2021.12.17 + [debug] youtube-dl version 2020.09.20 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/.github/ISSUE_TEMPLATE/2_site_support_request.md b/.github/ISSUE_TEMPLATE/2_site_support_request.md index 33b01ce7f..a4002603c 100644 --- a/.github/ISSUE_TEMPLATE/2_site_support_request.md +++ b/.github/ISSUE_TEMPLATE/2_site_support_request.md @@ -19,7 +19,7 @@ labels: 'site-support-request' - [ ] I'm reporting a new site support request -- [ ] I've verified that I'm running youtube-dl version **2021.12.17** +- [ ] I've verified that I'm running youtube-dl version **2020.09.20** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that none of provided URLs violate any copyrights - [ ] I've searched the bugtracker for similar site support requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/3_site_feature_request.md b/.github/ISSUE_TEMPLATE/3_site_feature_request.md index 285610cc7..3f8b6ce2e 100644 --- a/.github/ISSUE_TEMPLATE/3_site_feature_request.md +++ b/.github/ISSUE_TEMPLATE/3_site_feature_request.md @@ -18,13 +18,13 @@ title: '' - [ ] I'm reporting a site feature request -- [ ] I've verified that I'm running youtube-dl version **2021.12.17** +- [ ] I've verified that I'm running youtube-dl version **2020.09.20** - [ ] I've searched the bugtracker for similar site feature requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/4_bug_report.md b/.github/ISSUE_TEMPLATE/4_bug_report.md index af73525fb..d880c225a 100644 --- a/.github/ISSUE_TEMPLATE/4_bug_report.md +++ b/.github/ISSUE_TEMPLATE/4_bug_report.md @@ -18,7 +18,7 @@ title: '' - [ ] I'm reporting a broken site support issue -- [ ] I've verified that I'm running youtube-dl version **2021.12.17** +- [ ] I've verified that I'm running youtube-dl version **2020.09.20** - [ ] I've checked that all provided URLs are alive and playable in a browser - [ ] I've checked that all URLs and arguments with special characters are properly quoted or escaped - [ ] I've searched the bugtracker for similar bug reports including closed ones @@ -43,7 +43,7 @@ Add the `-v` flag to your command line you run youtube-dl with (`youtube-dl -v < [debug] User config: [] [debug] Command-line args: [u'-v', u'http://www.youtube.com/watch?v=BaW_jenozKcj'] [debug] Encodings: locale cp1251, fs mbcs, out cp866, pref cp1251 - [debug] youtube-dl version 2021.12.17 + [debug] youtube-dl version 2020.09.20 [debug] Python version 2.7.11 - Windows-2003Server-5.2.3790-SP2 [debug] exe versions: ffmpeg N-75573-g1d0487f, ffprobe N-75573-g1d0487f, rtmpdump 2.4 [debug] Proxy map: {} diff --git a/.github/ISSUE_TEMPLATE/5_feature_request.md b/.github/ISSUE_TEMPLATE/5_feature_request.md index 42c878b83..dd5fb5144 100644 --- a/.github/ISSUE_TEMPLATE/5_feature_request.md +++ b/.github/ISSUE_TEMPLATE/5_feature_request.md @@ -19,13 +19,13 @@ labels: 'request' - [ ] I'm reporting a feature request -- [ ] I've verified that I'm running youtube-dl version **2021.12.17** +- [ ] I've verified that I'm running youtube-dl version **2020.09.20** - [ ] I've searched the bugtracker for similar feature requests including closed ones diff --git a/.github/ISSUE_TEMPLATE/config.yml b/.github/ISSUE_TEMPLATE/config.yml deleted file mode 100644 index 3ba13e0ce..000000000 --- a/.github/ISSUE_TEMPLATE/config.yml +++ /dev/null @@ -1 +0,0 @@ -blank_issues_enabled: false diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 892cea0a3..e69b907d8 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -7,10 +7,8 @@ --- ### Before submitting a *pull request* make sure you have: +- [ ] At least skimmed through [adding new extractor tutorial](https://github.com/ytdl-org/youtube-dl#adding-support-for-a-new-site) and [youtube-dl coding conventions](https://github.com/ytdl-org/youtube-dl#youtube-dl-coding-conventions) sections - [ ] [Searched](https://github.com/ytdl-org/youtube-dl/search?q=is%3Apr&type=Issues) the bugtracker for similar pull requests -- [ ] Read [adding new extractor tutorial](https://github.com/ytdl-org/youtube-dl#adding-support-for-a-new-site) -- [ ] Read [youtube-dl coding conventions](https://github.com/ytdl-org/youtube-dl#youtube-dl-coding-conventions) and adjusted the code to meet them -- [ ] Covered the code with tests (note that PRs without tests will be REJECTED) - [ ] Checked the code with [flake8](https://pypi.python.org/pypi/flake8) ### In order to be accepted and merged into youtube-dl each piece of code must be in public domain or released under [Unlicense](http://unlicense.org/). Check one of the following options: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml deleted file mode 100644 index 8234e0ccb..000000000 --- a/.github/workflows/ci.yml +++ /dev/null @@ -1,482 +0,0 @@ -name: CI - -env: - all-cpython-versions: 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6, 3.7, 3.8, 3.9, 3.10, 3.11, 3.12 - main-cpython-versions: 2.7, 3.2, 3.5, 3.9, 3.11 - pypy-versions: pypy-2.7, pypy-3.6, pypy-3.7 - cpython-versions: main - test-set: core - # Python beta version to be built using pyenv before setup-python support - # Must also be included in all-cpython-versions - next: 3.13 - -on: - push: - # push inputs aren't known to GitHub - inputs: - cpython-versions: - type: string - default: all - test-set: - type: string - default: core - pull_request: - # pull_request inputs aren't known to GitHub - inputs: - cpython-versions: - type: string - default: main - test-set: - type: string - default: both - workflow_dispatch: - inputs: - cpython-versions: - type: choice - description: CPython versions (main = 2.7, 3.2, 3.5, 3.9, 3.11) - options: - - all - - main - required: true - default: main - test-set: - type: choice - description: core, download - options: - - both - - core - - download - required: true - default: both - -permissions: - contents: read - -jobs: - select: - name: Select tests from inputs - runs-on: ubuntu-latest - outputs: - cpython-versions: ${{ steps.run.outputs.cpython-versions }} - test-set: ${{ steps.run.outputs.test-set }} - own-pip-versions: ${{ steps.run.outputs.own-pip-versions }} - steps: - # push and pull_request inputs aren't known to GitHub (pt3) - - name: Set push defaults - if: ${{ github.event_name == 'push' }} - env: - cpython-versions: all - test-set: core - run: | - echo "cpython-versions=${{env.cpython-versions}}" >> "$GITHUB_ENV" - echo "test_set=${{env.test_set}}" >> "$GITHUB_ENV" - - name: Get pull_request inputs - if: ${{ github.event_name == 'pull_request' }} - env: - cpython-versions: main - test-set: both - run: | - echo "cpython-versions=${{env.cpython-versions}}" >> "$GITHUB_ENV" - echo "test_set=${{env.test_set}}" >> "$GITHUB_ENV" - - name: Make version array - id: run - run: | - # Make a JSON Array from comma/space-separated string (no extra escaping) - json_list() { \ - ret=""; IFS="${IFS},"; set -- $*; \ - for a in "$@"; do \ - ret=$(printf '%s"%s"' "${ret}${ret:+, }" "$a"); \ - done; \ - printf '[%s]' "$ret"; } - tests="${{ inputs.test-set || env.test-set }}" - [ $tests = both ] && tests="core download" - printf 'test-set=%s\n' "$(json_list $tests)" >> "$GITHUB_OUTPUT" - versions="${{ inputs.cpython-versions || env.cpython-versions }}" - if [ "$versions" = all ]; then \ - versions="${{ env.all-cpython-versions }}"; else \ - versions="${{ env.main-cpython-versions }}"; \ - fi - printf 'cpython-versions=%s\n' \ - "$(json_list ${versions}${versions:+, }${{ env.pypy-versions }})" >> "$GITHUB_OUTPUT" - # versions with a special get-pip.py in a per-version subdirectory - printf 'own-pip-versions=%s\n' \ - "$(json_list 2.6, 2.7, 3.2, 3.3, 3.4, 3.5, 3.6)" >> "$GITHUB_OUTPUT" - - tests: - name: Run tests - needs: select - permissions: - contents: read - packages: write - runs-on: ${{ matrix.os }} - env: - PIP: python -m pip - PIP_DISABLE_PIP_VERSION_CHECK: true - PIP_NO_PYTHON_VERSION_WARNING: true - strategy: - fail-fast: true - matrix: - os: [ubuntu-22.04] - python-version: ${{ fromJSON(needs.select.outputs.cpython-versions) }} - python-impl: [cpython] - ytdl-test-set: ${{ fromJSON(needs.select.outputs.test-set) }} - run-tests-ext: [sh] - include: - - os: windows-2019 - python-version: 3.4 - python-impl: cpython - ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }} - run-tests-ext: bat - - os: windows-2019 - python-version: 3.4 - python-impl: cpython - ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download' || 'nodownload' }} - run-tests-ext: bat - # jython - - os: ubuntu-22.04 - python-version: 2.7 - python-impl: jython - ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'core') && 'core' || 'nocore' }} - run-tests-ext: sh - - os: ubuntu-22.04 - python-version: 2.7 - python-impl: jython - ytdl-test-set: ${{ contains(needs.select.outputs.test-set, 'download') && 'download' || 'nodownload' }} - run-tests-ext: sh - steps: - - name: Prepare Linux - if: ${{ startswith(matrix.os, 'ubuntu') }} - shell: bash - run: | - # apt in runner, if needed, may not be up-to-date - sudo apt-get update - - name: Checkout - uses: actions/checkout@v3 - #-------- Python 3 ----- - - name: Set up supported Python ${{ matrix.python-version }} - id: setup-python - if: ${{ matrix.python-impl == 'cpython' && matrix.python-version != '2.6' && matrix.python-version != '2.7' && matrix.python-version != env.next }} - # wrap broken actions/setup-python@v4 - # NB may run apt-get install in Linux - uses: ytdl-org/setup-python@v1 - env: - # Temporary (?) workaround for Python 3.5 failures - May 2024 - PIP_TRUSTED_HOST: "pypi.python.org pypi.org files.pythonhosted.org" - with: - python-version: ${{ matrix.python-version }} - cache-build: true - allow-build: info - - name: Locate supported Python ${{ matrix.python-version }} - if: ${{ env.pythonLocation }} - shell: bash - run: | - echo "PYTHONHOME=${pythonLocation}" >> "$GITHUB_ENV" - export expected="${{ steps.setup-python.outputs.python-path }}" - dirname() { printf '%s\n' \ - 'import os, sys' \ - 'print(os.path.dirname(sys.argv[1]))' \ - | ${expected} - "$1"; } - expd="$(dirname "$expected")" - export python="$(command -v python)" - [ "$expd" = "$(dirname "$python")" ] || echo "PATH=$expd:${PATH}" >> "$GITHUB_ENV" - [ -x "$python" ] || printf '%s\n' \ - 'import os' \ - 'exp = os.environ["expected"]' \ - 'python = os.environ["python"]' \ - 'exps = os.path.split(exp)' \ - 'if python and (os.path.dirname(python) == exp[0]):' \ - ' exit(0)' \ - 'exps[1] = "python" + os.path.splitext(exps[1])[1]' \ - 'python = os.path.join(*exps)' \ - 'try:' \ - ' os.symlink(exp, python)' \ - 'except AttributeError:' \ - ' os.rename(exp, python)' \ - | ${expected} - - printf '%s\n' \ - 'import sys' \ - 'print(sys.path)' \ - | ${expected} - - #-------- Python next (was 3.12) - - - name: Set up CPython 3.next environment - if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next }} - shell: bash - run: | - PYENV_ROOT=$HOME/.local/share/pyenv - echo "PYENV_ROOT=${PYENV_ROOT}" >> "$GITHUB_ENV" - - name: Cache Python 3.next - id: cachenext - if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next }} - uses: actions/cache@v3 - with: - key: python-${{ env.next }} - path: | - ${{ env.PYENV_ROOT }} - - name: Build and set up Python 3.next - if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next && ! steps.cachenext.outputs.cache-hit }} - # dl and build locally - shell: bash - run: | - # Install build environment - sudo apt-get install -y build-essential llvm libssl-dev tk-dev \ - libncursesw5-dev libreadline-dev libsqlite3-dev \ - libffi-dev xz-utils zlib1g-dev libbz2-dev liblzma-dev - # Download PyEnv from its GitHub repository. - export PYENV_ROOT=${{ env.PYENV_ROOT }} - export PATH=$PYENV_ROOT/bin:$PATH - git clone "https://github.com/pyenv/pyenv.git" "$PYENV_ROOT" - pyenv install ${{ env.next }} - - name: Locate Python 3.next - if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == env.next }} - shell: bash - run: | - PYTHONHOME="$(echo "${{ env.PYENV_ROOT }}/versions/${{ env.next }}."*)" - test -n "$PYTHONHOME" - echo "PYTHONHOME=$PYTHONHOME" >> "$GITHUB_ENV" - echo "PATH=${PYTHONHOME}/bin:$PATH" >> "$GITHUB_ENV" - #-------- Python 2.7 -- - - name: Set up Python 2.7 - if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.7' }} - # install 2.7 - shell: bash - run: | - # Ubuntu 22.04 no longer has python-is-python2: fetch it - curl -L "http://launchpadlibrarian.net/474693132/python-is-python2_2.7.17-4_all.deb" -o python-is-python2.deb - sudo apt-get install -y python2 - sudo dpkg --force-breaks -i python-is-python2.deb - echo "PYTHONHOME=/usr" >> "$GITHUB_ENV" - #-------- Python 2.6 -- - - name: Set up Python 2.6 environment - if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' }} - shell: bash - run: | - openssl_name=openssl-1.0.2u - echo "openssl_name=${openssl_name}" >> "$GITHUB_ENV" - openssl_dir=$HOME/.local/opt/$openssl_name - echo "openssl_dir=${openssl_dir}" >> "$GITHUB_ENV" - PYENV_ROOT=$HOME/.local/share/pyenv - echo "PYENV_ROOT=${PYENV_ROOT}" >> "$GITHUB_ENV" - sudo apt-get install -y openssl ca-certificates - - name: Cache Python 2.6 - id: cache26 - if: ${{ matrix.python-version == '2.6' }} - uses: actions/cache@v3 - with: - key: python-2.6.9 - path: | - ${{ env.openssl_dir }} - ${{ env.PYENV_ROOT }} - - name: Build and set up Python 2.6 - if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' && ! steps.cache26.outputs.cache-hit }} - # dl and build locally - shell: bash - run: | - # Install build environment - sudo apt-get install -y build-essential llvm libssl-dev tk-dev \ - libncursesw5-dev libreadline-dev libsqlite3-dev \ - libffi-dev xz-utils zlib1g-dev libbz2-dev liblzma-dev - # Download and install OpenSSL 1.0.2, back in time - openssl_name=${{ env.openssl_name }} - openssl_targz=${openssl_name}.tar.gz - openssl_dir=${{ env.openssl_dir }} - openssl_inc=$openssl_dir/include - openssl_lib=$openssl_dir/lib - openssl_ssl=$openssl_dir/ssl - curl -L "https://www.openssl.org/source/$openssl_targz" -o $openssl_targz - tar -xf $openssl_targz - ( cd $openssl_name; \ - ./config --prefix=$openssl_dir --openssldir=${openssl_dir}/ssl \ - --libdir=lib -Wl,-rpath=${openssl_dir}/lib shared zlib-dynamic && \ - make && \ - make install ) - rm -rf $openssl_name - rmdir $openssl_ssl/certs && ln -s /etc/ssl/certs $openssl_ssl/certs - # Download PyEnv from its GitHub repository. - export PYENV_ROOT=${{ env.PYENV_ROOT }} - export PATH=$PYENV_ROOT/bin:$PATH - git clone "https://github.com/pyenv/pyenv.git" "$PYENV_ROOT" - # Prevent pyenv build trying (and failing) to update pip - export GET_PIP=get-pip-2.6.py - echo 'import sys; sys.exit(0)' > ${GET_PIP} - GET_PIP=$(realpath $GET_PIP) - # Build and install Python - export CFLAGS="-I$openssl_inc" - export LDFLAGS="-L$openssl_lib" - export LD_LIBRARY_PATH="$openssl_lib" - pyenv install 2.6.9 - - name: Locate Python 2.6 - if: ${{ matrix.python-impl == 'cpython' && matrix.python-version == '2.6' }} - shell: bash - run: | - PYTHONHOME="${{ env.PYENV_ROOT }}/versions/2.6.9" - echo "PYTHONHOME=$PYTHONHOME" >> "$GITHUB_ENV" - echo "PATH=${PYTHONHOME}/bin:$PATH" >> "$GITHUB_ENV" - echo "LD_LIBRARY_PATH=${{ env.openssl_dir }}/lib${LD_LIBRARY_PATH:+:}${LD_LIBRARY_PATH}" >> "$GITHUB_ENV" - #-------- Jython ------ - - name: Set up Java 8 - if: ${{ matrix.python-impl == 'jython' }} - uses: actions/setup-java@v3 - with: - java-version: 8 - distribution: 'zulu' - - name: Setup Jython environment - if: ${{ matrix.python-impl == 'jython' }} - shell: bash - run: | - echo "JYTHON_ROOT=${HOME}/jython" >> "$GITHUB_ENV" - echo "PIP=pip" >> "$GITHUB_ENV" - - name: Cache Jython - id: cachejy - if: ${{ matrix.python-impl == 'jython' && matrix.python-version == '2.7' }} - uses: actions/cache@v3 - with: - # 2.7.3 now available, may solve SNI issue - key: jython-2.7.1 - path: | - ${{ env.JYTHON_ROOT }} - - name: Install Jython - if: ${{ matrix.python-impl == 'jython' && matrix.python-version == '2.7' && ! steps.cachejy.outputs.cache-hit }} - shell: bash - run: | - JYTHON_ROOT="${{ env.JYTHON_ROOT }}" - curl -L "https://repo1.maven.org/maven2/org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar" -o jython-installer.jar - java -jar jython-installer.jar -s -d "${JYTHON_ROOT}" - echo "${JYTHON_ROOT}/bin" >> "$GITHUB_PATH" - - name: Set up cached Jython - if: ${{ steps.cachejy.outputs.cache-hit }} - shell: bash - run: | - JYTHON_ROOT="${{ env.JYTHON_ROOT }}" - echo "${JYTHON_ROOT}/bin" >> $GITHUB_PATH - - name: Install supporting Python 2.7 if possible - if: ${{ steps.cachejy.outputs.cache-hit }} - shell: bash - run: | - sudo apt-get install -y python2.7 || true - #-------- pip --------- - - name: Set up supported Python ${{ matrix.python-version }} pip - if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || matrix.python-version == '2.7' }} - # This step may run in either Linux or Windows - shell: bash - run: | - echo "$PATH" - echo "$PYTHONHOME" - # curl is available on both Windows and Linux, -L follows redirects, -O gets name - python -m ensurepip || python -m pip --version || { \ - get_pip="${{ contains(needs.select.outputs.own-pip-versions, matrix.python-version) && format('{0}/', matrix.python-version) || '' }}"; \ - curl -L -O "https://bootstrap.pypa.io/pip/${get_pip}get-pip.py"; \ - python get-pip.py; } - - name: Set up Python 2.6 pip - if: ${{ matrix.python-version == '2.6' }} - shell: bash - run: | - python -m pip --version || { \ - curl -L -O "https://bootstrap.pypa.io/pip/2.6/get-pip.py"; \ - curl -L -O "https://files.pythonhosted.org/packages/ac/95/a05b56bb975efa78d3557efa36acaf9cf5d2fd0ee0062060493687432e03/pip-9.0.3-py2.py3-none-any.whl"; \ - python get-pip.py --no-setuptools --no-wheel pip-9.0.3-py2.py3-none-any.whl; } - # work-around to invoke pip module on 2.6: https://bugs.python.org/issue2751 - echo "PIP=python -m pip.__main__" >> "$GITHUB_ENV" - - name: Set up other Python ${{ matrix.python-version }} pip - if: ${{ matrix.python-version == '3.2' && steps.setup-python.outputs.python-path }} - shell: bash - run: | - python -m pip --version || { \ - curl -L -O "https://bootstrap.pypa.io/pip/3.2/get-pip.py"; \ - curl -L -O "https://files.pythonhosted.org/packages/b2/d0/cd115fe345dd6f07ec1c780020a7dfe74966fceeb171e0f20d1d4905b0b7/pip-7.1.2-py2.py3-none-any.whl"; \ - python get-pip.py --no-setuptools --no-wheel pip-7.1.2-py2.py3-none-any.whl; } - #-------- unittest ---- - - name: Upgrade Unittest for Python 2.6 - if: ${{ matrix.python-version == '2.6' }} - shell: bash - run: | - # Work around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb) - $PIP -qq show unittest2 || { \ - for u in "65/26/32b8464df2a97e6dd1b656ed26b2c194606c16fe163c695a992b36c11cdf/six-1.13.0-py2.py3-none-any.whl" \ - "f2/94/3af39d34be01a24a6e65433d19e107099374224905f1e0cc6bbe1fd22a2f/argparse-1.4.0-py2.py3-none-any.whl" \ - "c7/a3/c5da2a44c85bfbb6eebcfc1dde24933f8704441b98fdde6528f4831757a6/linecache2-1.0.0-py2.py3-none-any.whl" \ - "17/0a/6ac05a3723017a967193456a2efa0aa9ac4b51456891af1e2353bb9de21e/traceback2-1.4.0-py2.py3-none-any.whl" \ - "72/20/7f0f433060a962200b7272b8c12ba90ef5b903e218174301d0abfd523813/unittest2-1.1.0-py2.py3-none-any.whl"; do \ - curl -L -O "https://files.pythonhosted.org/packages/${u}"; \ - $PIP install ${u##*/}; \ - done; } - # make tests use unittest2 - for test in ./test/test_*.py ./test/helper.py; do - sed -r -i -e '/^import unittest$/s/test/test2 as unittest/' "$test" - done - #-------- nose -------- - - name: Install nose for Python ${{ matrix.python-version }} - if: ${{ (matrix.python-version != '3.2' && steps.setup-python.outputs.python-path) || (matrix.python-impl == 'cpython' && (matrix.python-version == '2.7' || matrix.python-version == env.next)) }} - shell: bash - run: | - echo "$PATH" - echo "$PYTHONHOME" - # Use PyNose for recent Pythons instead of Nose - py3ver="${{ matrix.python-version }}" - py3ver=${py3ver#3.} - [ "$py3ver" != "${{ matrix.python-version }}" ] && py3ver=${py3ver%.*} || py3ver=0 - [ "$py3ver" -ge 9 ] && nose=pynose || nose=nose - $PIP -qq show $nose || $PIP install $nose - - name: Install nose for other Python 2 - if: ${{ matrix.python-impl == 'jython' || (matrix.python-impl == 'cpython' && matrix.python-version == '2.6') }} - shell: bash - run: | - # Work around deprecation of support for non-SNI clients at PyPI CDN (see https://status.python.org/incidents/hzmjhqsdjqgb) - $PIP -qq show nose || { \ - curl -L -O "https://files.pythonhosted.org/packages/99/4f/13fb671119e65c4dce97c60e67d3fd9e6f7f809f2b307e2611f4701205cb/nose-1.3.7-py2-none-any.whl"; \ - $PIP install nose-1.3.7-py2-none-any.whl; } - - name: Install nose for other Python 3 - if: ${{ matrix.python-version == '3.2' && steps.setup-python.outputs.python-path }} - shell: bash - run: | - $PIP -qq show nose || { \ - curl -L -O "https://files.pythonhosted.org/packages/15/d8/dd071918c040f50fa1cf80da16423af51ff8ce4a0f2399b7bf8de45ac3d9/nose-1.3.7-py3-none-any.whl"; \ - $PIP install nose-1.3.7-py3-none-any.whl; } - - name: Set up nosetest test - if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }} - shell: bash - run: | - # set PYTHON_VER - PYTHON_VER=${{ matrix.python-version }} - [ "${PYTHON_VER#*-}" != "$PYTHON_VER" ] || PYTHON_VER="${{ matrix.python-impl }}-${PYTHON_VER}" - echo "PYTHON_VER=$PYTHON_VER" >> "$GITHUB_ENV" - echo "PYTHON_IMPL=${{ matrix.python-impl }}" >> "$GITHUB_ENV" - # define a test to validate the Python version used by nosetests - printf '%s\n' \ - 'from __future__ import unicode_literals' \ - 'import sys, os, platform' \ - 'try:' \ - ' import unittest2 as unittest' \ - 'except ImportError:' \ - ' import unittest' \ - 'class TestPython(unittest.TestCase):' \ - ' def setUp(self):' \ - ' self.ver = os.environ["PYTHON_VER"].split("-")' \ - ' def test_python_ver(self):' \ - ' self.assertEqual(["%d" % v for v in sys.version_info[:2]], self.ver[-1].split(".")[:2])' \ - ' self.assertTrue(sys.version.startswith(self.ver[-1]))' \ - ' self.assertIn(self.ver[0], ",".join((sys.version, platform.python_implementation())).lower())' \ - ' def test_python_impl(self):' \ - ' self.assertIn(platform.python_implementation().lower(), (os.environ["PYTHON_IMPL"], self.ver[0]))' \ - > test/test_python.py - #-------- TESTS ------- - - name: Run tests - if: ${{ contains(needs.select.outputs.test-set, matrix.ytdl-test-set ) }} - continue-on-error: ${{ matrix.ytdl-test-set == 'download' || matrix.python-impl == 'jython' }} - env: - YTDL_TEST_SET: ${{ matrix.ytdl-test-set }} - run: | - ./devscripts/run_tests.${{ matrix.run-tests-ext }} - flake8: - name: Linter - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - - name: Set up Python - uses: actions/setup-python@v4 - with: - python-version: 3.9 - - name: Install flake8 - run: pip install flake8 - - name: Run flake8 - run: flake8 . - diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 000000000..51afd469a --- /dev/null +++ b/.travis.yml @@ -0,0 +1,50 @@ +language: python +python: + - "2.6" + - "2.7" + - "3.2" + - "3.3" + - "3.4" + - "3.5" + - "3.6" + - "pypy" + - "pypy3" +dist: trusty +env: + - YTDL_TEST_SET=core + - YTDL_TEST_SET=download +jobs: + include: + - python: 3.7 + dist: xenial + env: YTDL_TEST_SET=core + - python: 3.7 + dist: xenial + env: YTDL_TEST_SET=download + - python: 3.8 + dist: xenial + env: YTDL_TEST_SET=core + - python: 3.8 + dist: xenial + env: YTDL_TEST_SET=download + - python: 3.8-dev + dist: xenial + env: YTDL_TEST_SET=core + - python: 3.8-dev + dist: xenial + env: YTDL_TEST_SET=download + - env: JYTHON=true; YTDL_TEST_SET=core + - env: JYTHON=true; YTDL_TEST_SET=download + - name: flake8 + python: 3.8 + dist: xenial + install: pip install flake8 + script: flake8 . + fast_finish: true + allow_failures: + - env: YTDL_TEST_SET=download + - env: JYTHON=true; YTDL_TEST_SET=core + - env: JYTHON=true; YTDL_TEST_SET=download +before_install: + - if [ "$JYTHON" == "true" ]; then ./devscripts/install_jython.sh; export PATH="$HOME/jython/bin:$PATH"; fi +script: ./devscripts/run_tests.sh diff --git a/AUTHORS b/AUTHORS index 4a6d7dacd..b507cb8df 100644 --- a/AUTHORS +++ b/AUTHORS @@ -246,4 +246,3 @@ Enes Solak Nathan Rossi Thomas van der Berg Luca Cherubin -Adrian Heine \ No newline at end of file diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index ff40cef78..58ab3a4b8 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -150,7 +150,7 @@ After you have ensured this site is distributing its content legally, you can fo # TODO more properties (see youtube_dl/extractor/common.py) } ``` -5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py). This makes the extractor available for use, as long as the class ends with `IE`. +5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py). 6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in. 7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want. 8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart): diff --git a/ChangeLog b/ChangeLog index 658864282..9b52b7bd2 100644 --- a/ChangeLog +++ b/ChangeLog @@ -1,876 +1,3 @@ -version 2021.12.17 - -Core -* [postprocessor/ffmpeg] Show ffmpeg output on error (#22680, #29336) - -Extractors -* [youtube] Update signature function patterns (#30363, #30366) -* [peertube] Only call description endpoint if necessary (#29383) -* [periscope] Pass referer to HLS requests (#29419) -- [liveleak] Remove extractor (#17625, #24222, #29331) -+ [pornhub] Add support for pornhubthbh7ap3u.onion -* [pornhub] Detect geo restriction -* [pornhub] Dismiss tbr extracted from download URLs (#28927) -* [curiositystream:collection] Extend _VALID_URL (#26326, #29117) -* [youtube] Make get_video_info processing more robust (#29333) -* [youtube] Workaround for get_video_info request (#29333) -* [bilibili] Strip uploader name (#29202) -* [youtube] Update invidious instance list (#29281) -* [umg:de] Update GraphQL API URL (#29304) -* [nrk] Switch psapi URL to https (#29344) -+ [egghead] Add support for app.egghead.io (#28404, #29303) -* [appleconnect] Fix extraction (#29208) -+ [orf:tvthek] Add support for MPD formats (#28672, #29236) - - -version 2021.06.06 - -Extractors -* [facebook] Improve login required detection -* [youporn] Fix formats and view count extraction (#29216) -* [orf:tvthek] Fix thumbnails extraction (#29217) -* [formula1] Fix extraction (#29206) -* [ard] Relax URL regular expression and fix video ids (#22724, #29091) -+ [ustream] Detect https embeds (#29133) -* [ted] Prefer own formats over external sources (#29142) -* [twitch:clips] Improve extraction (#29149) -+ [twitch:clips] Add access token query to download URLs (#29136) -* [youtube] Fix get_video_info request (#29086, #29165) -* [vimeo] Fix vimeo pro embed extraction (#29126) -* [redbulltv] Fix embed data extraction (#28770) -* [shahid] Relax URL regular expression (#28772, #28930) - - -version 2021.05.16 - -Core -* [options] Fix thumbnail option group name (#29042) -* [YoutubeDL] Improve extract_info doc (#28946) - -Extractors -+ [playstuff] Add support for play.stuff.co.nz (#28901, #28931) -* [eroprofile] Fix extraction (#23200, #23626, #29008) -+ [vivo] Add support for vivo.st (#29009) -+ [generic] Add support for og:audio (#28311, #29015) -* [phoenix] Fix extraction (#29057) -+ [generic] Add support for sibnet embeds -+ [vk] Add support for sibnet embeds (#9500) -+ [generic] Add Referer header for direct videojs download URLs (#2879, - #20217, #29053) -* [orf:radio] Switch download URLs to HTTPS (#29012, #29046) -- [blinkx] Remove extractor (#28941) -* [medaltv] Relax URL regular expression (#28884) -+ [funimation] Add support for optional lang code in URLs (#28950) -+ [gdcvault] Add support for HTML5 videos -* [dispeak] Improve FLV extraction (#13513, #28970) -* [kaltura] Improve iframe extraction (#28969) -* [kaltura] Make embed code alternatives actually work -* [cda] Improve extraction (#28709, #28937) -* [twitter] Improve formats extraction from vmap URL (#28909) -* [xtube] Fix formats extraction (#28870) -* [svtplay] Improve extraction (#28507, #28876) -* [tv2dk] Fix extraction (#28888) - - -version 2021.04.26 - -Extractors -+ [xfileshare] Add support for wolfstream.tv (#28858) -* [francetvinfo] Improve video id extraction (#28792) -* [medaltv] Fix extraction (#28807) -* [tver] Redirect all downloads to Brightcove (#28849) -* [go] Improve video id extraction (#25207, #25216, #26058) -* [youtube] Fix lazy extractors (#28780) -+ [bbc] Extract description and timestamp from __INITIAL_DATA__ (#28774) -* [cbsnews] Fix extraction for python <3.6 (#23359) - - -version 2021.04.17 - -Core -+ [utils] Add support for experimental HTTP response status code - 308 Permanent Redirect (#27877, #28768) - -Extractors -+ [lbry] Add support for HLS videos (#27877, #28768) -* [youtube] Fix stretched ratio calculation -* [youtube] Improve stretch extraction (#28769) -* [youtube:tab] Improve grid extraction (#28725) -+ [youtube:tab] Detect series playlist on playlists page (#28723) -+ [youtube] Add more invidious instances (#28706) -* [pluralsight] Extend anti-throttling timeout (#28712) -* [youtube] Improve URL to extractor routing (#27572, #28335, #28742) -+ [maoritv] Add support for maoritelevision.com (#24552) -+ [youtube:tab] Pass innertube context and x-goog-visitor-id header along with - continuation requests (#28702) -* [mtv] Fix Viacom A/B Testing Video Player extraction (#28703) -+ [pornhub] Extract DASH and HLS formats from get_media end point (#28698) -* [cbssports] Fix extraction (#28682) -* [jamendo] Fix track extraction (#28686) -* [curiositystream] Fix format extraction (#26845, #28668) - - -version 2021.04.07 - -Core -* [extractor/common] Use compat_cookies_SimpleCookie for _get_cookies -+ [compat] Introduce compat_cookies_SimpleCookie -* [extractor/common] Improve JSON-LD author extraction -* [extractor/common] Fix _get_cookies on python 2 (#20673, #23256, #20326, - #28640) - -Extractors -* [youtube] Fix extraction of videos with restricted location (#28685) -+ [line] Add support for live.line.me (#17205, #28658) -* [vimeo] Improve extraction (#28591) -* [youku] Update ccode (#17852, #28447, #28460, #28648) -* [youtube] Prefer direct entry metadata over entry metadata from playlist - (#28619, #28636) -* [screencastomatic] Fix extraction (#11976, #24489) -+ [palcomp3] Add support for palcomp3.com (#13120) -+ [arnes] Add support for video.arnes.si (#28483) -+ [youtube:tab] Add support for hashtags (#28308) - - -version 2021.04.01 - -Extractors -* [youtube] Setup CONSENT cookie when needed (#28604) -* [vimeo] Fix password protected review extraction (#27591) -* [youtube] Improve age-restricted video extraction (#28578) - - -version 2021.03.31 - -Extractors -* [vlive] Fix inkey request (#28589) -* [francetvinfo] Improve video id extraction (#28584) -+ [instagram] Extract duration (#28469) -* [instagram] Improve title extraction (#28469) -+ [sbs] Add support for ondemand watch URLs (#28566) -* [youtube] Fix video's channel extraction (#28562) -* [picarto] Fix live stream extraction (#28532) -* [vimeo] Fix unlisted video extraction (#28414) -* [youtube:tab] Fix playlist/community continuation items extraction (#28266) -* [ard] Improve clip id extraction (#22724, #28528) - - -version 2021.03.25 - -Extractors -+ [zoom] Add support for zoom.us (#16597, #27002, #28531) -* [bbc] Fix BBC IPlayer Episodes/Group extraction (#28360) -* [youtube] Fix default value for youtube_include_dash_manifest (#28523) -* [zingmp3] Fix extraction (#11589, #16409, #16968, #27205) -+ [vgtv] Add support for new tv.aftonbladet.se URL schema (#28514) -+ [tiktok] Detect private videos (#28453) -* [vimeo:album] Fix extraction for albums with number of videos multiple - to page size (#28486) -* [vvvvid] Fix kenc format extraction (#28473) -* [mlb] Fix video extraction (#21241) -* [svtplay] Improve extraction (#28448) -* [applepodcasts] Fix extraction (#28445) -* [rtve] Improve extraction - + Extract all formats - * Fix RTVE Infantil extraction (#24851) - + Extract is_live and series - - -version 2021.03.14 - -Core -+ Introduce release_timestamp meta field (#28386) - -Extractors -+ [southpark] Add support for southparkstudios.com (#28413) -* [southpark] Fix extraction (#26763, #28413) -* [sportdeutschland] Fix extraction (#21856, #28425) -* [pinterest] Reduce the number of HLS format requests -* [peertube] Improve thumbnail extraction (#28419) -* [tver] Improve title extraction (#28418) -* [fujitv] Fix HLS formats extension (#28416) -* [shahid] Fix format extraction (#28383) -+ [lbry] Add support for channel filters (#28385) -+ [bandcamp] Extract release timestamp -+ [lbry] Extract release timestamp (#28386) -* [pornhub] Detect flagged videos -+ [pornhub] Extract formats from get_media end point (#28395) -* [bilibili] Fix video info extraction (#28341) -+ [cbs] Add support for Paramount+ (#28342) -+ [trovo] Add Origin header to VOD formats (#28346) -* [voxmedia] Fix volume embed extraction (#28338) - - -version 2021.03.03 - -Extractors -* [youtube:tab] Switch continuation to browse API (#28289, #28327) -* [9c9media] Fix extraction for videos with multiple ContentPackages (#28309) -+ [bbc] Add support for BBC Reel videos (#21870, #23660, #28268) - - -version 2021.03.02 - -Extractors -* [zdf] Rework extractors (#11606, #13473, #17354, #21185, #26711, #27068, - #27930, #28198, #28199, #28274) - * Generalize cross-extractor video ids for zdf based extractors - * Improve extraction - * Fix 3sat and phoenix -* [stretchinternet] Fix extraction (#28297) -* [urplay] Fix episode data extraction (#28292) -+ [bandaichannel] Add support for b-ch.com (#21404) -* [srgssr] Improve extraction (#14717, #14725, #27231, #28238) - + Extract subtitle - * Fix extraction for new videos - * Update srf download domains -* [vvvvid] Reduce season request payload size -+ [vvvvid] Extract series sublists playlist title (#27601, #27618) -+ [dplay] Extract Ad-Free uplynk URLs (#28160) -+ [wat] Detect DRM protected videos (#27958) -* [tf1] Improve extraction (#27980, #28040) -* [tmz] Fix and improve extraction (#24603, #24687, 28211) -+ [gedidigital] Add support for Gedi group sites (#7347, #26946) -* [youtube] Fix get_video_info request - - -version 2021.02.22 - -Core -+ [postprocessor/embedthumbnail] Recognize atomicparsley binary in lowercase - (#28112) - -Extractors -* [apa] Fix and improve extraction (#27750) -+ [youporn] Extract duration (#28019) -+ [peertube] Add support for canard.tube (#28190) -* [youtube] Fixup m4a_dash formats (#28165) -+ [samplefocus] Add support for samplefocus.com (#27763) -+ [vimeo] Add support for unlisted video source format extraction -* [viki] Improve extraction (#26522, #28203) - * Extract uploader URL and episode number - * Report login required error - + Extract 480p formats - * Fix API v4 calls -* [ninegag] Unescape title (#28201) -* [youtube] Improve URL regular expression (#28193) -+ [youtube] Add support for redirect.invidious.io (#28193) -+ [dplay] Add support for de.hgtv.com (#28182) -+ [dplay] Add support for discoveryplus.com (#24698) -+ [simplecast] Add support for simplecast.com (#24107) -* [youtube] Fix uploader extraction in flat playlist mode (#28045) -* [yandexmusic:playlist] Request missing tracks in chunks (#27355, #28184) -+ [storyfire] Add support for storyfire.com (#25628, #26349) -+ [zhihu] Add support for zhihu.com (#28177) -* [youtube] Fix controversial videos when authenticated with cookies (#28174) -* [ccma] Fix timestamp parsing in python 2 -+ [videopress] Add support for video.wordpress.com -* [kakao] Improve info extraction and detect geo restriction (#26577) -* [xboxclips] Fix extraction (#27151) -* [ard] Improve formats extraction (#28155) -+ [canvas] Add support for dagelijksekost.een.be (#28119) - - -version 2021.02.10 - -Extractors -* [youtube:tab] Improve grid continuation extraction (#28130) -* [ign] Fix extraction (#24771) -+ [xhamster] Extract format filesize -+ [xhamster] Extract formats from xplayer settings (#28114) -+ [youtube] Add support phone/tablet JS player (#26424) -* [archiveorg] Fix and improve extraction (#21330, #23586, #25277, #26780, - #27109, #27236, #28063) -+ [cda] Detect geo restricted videos (#28106) -* [urplay] Fix extraction (#28073, #28074) -* [youtube] Fix release date extraction (#28094) -+ [youtube] Extract abr and vbr (#28100) -* [youtube] Skip OTF formats (#28070) - - -version 2021.02.04.1 - -Extractors -* [youtube] Prefer DASH formats (#28070) -* [azmedien] Fix extraction (#28064) - - -version 2021.02.04 - -Extractors -* [pornhub] Implement lazy playlist extraction -* [svtplay] Fix video id extraction (#28058) -+ [pornhub] Add support for authentication (#18797, #21416, #24294) -* [pornhub:user] Improve paging -+ [pornhub:user] Add support for URLs unavailable via /videos page (#27853) -+ [bravotv] Add support for oxygen.com (#13357, #22500) -+ [youtube] Pass embed URL to get_video_info request -* [ccma] Improve metadata extraction (#27994) - + Extract age limit, alt title, categories, series and episode number - * Fix timestamp multiple subtitles extraction -* [egghead] Update API domain (#28038) -- [vidzi] Remove extractor (#12629) -* [vidio] Improve metadata extraction -* [youtube] Improve subtitles extraction -* [youtube] Fix chapter extraction fallback -* [youtube] Rewrite extractor - * Improve format sorting - * Remove unused code - * Fix series metadata extraction - * Fix trailer video extraction - * Improve error reporting - + Extract video location -+ [vvvvid] Add support for youtube embeds (#27825) -* [googledrive] Report download page errors (#28005) -* [vlive] Fix error message decoding for python 2 (#28004) -* [youtube] Improve DASH formats file size extraction -* [cda] Improve birth validation detection (#14022, #27929) -+ [awaan] Extract uploader id (#27963) -+ [medialaan] Add support DPG Media MyChannels based websites (#14871, #15597, - #16106, #16489) -* [abcnews] Fix extraction (#12394, #27920) -* [AMP] Fix upload date and timestamp extraction (#27970) -* [tv4] Relax URL regular expression (#27964) -+ [tv2] Add support for mtvuutiset.fi (#27744) -* [adn] Improve login warning reporting -* [zype] Fix uplynk id extraction (#27956) -+ [adn] Add support for authentication (#17091, #27841, #27937) - - -version 2021.01.24.1 - -Core -* Introduce --output-na-placeholder (#27896) - -Extractors -* [franceculture] Make thumbnail optional (#18807) -* [franceculture] Fix extraction (#27891, #27903) -* [njpwworld] Fix extraction (#27890) -* [comedycentral] Fix extraction (#27905) -* [wat] Fix format extraction (#27901) -+ [americastestkitchen:season] Add support for seasons (#27861) -+ [trovo] Add support for trovo.live (#26125) -+ [aol] Add support for yahoo videos (#26650) -* [yahoo] Fix single video extraction -* [lbry] Unescape lbry URI (#27872) -* [9gag] Fix and improve extraction (#23022) -* [americastestkitchen] Improve metadata extraction for ATK episodes (#27860) -* [aljazeera] Fix extraction (#20911, #27779) -+ [minds] Add support for minds.com (#17934) -* [ard] Fix title and description extraction (#27761) -+ [spotify] Add support for Spotify Podcasts (#27443) - - -version 2021.01.16 - -Core -* [YoutubeDL] Protect from infinite recursion due to recursively nested - playlists (#27833) -* [YoutubeDL] Ignore failure to create existing directory (#27811) -* [YoutubeDL] Raise syntax error for format selection expressions with multiple - + operators (#27803) - -Extractors -+ [animeondemand] Add support for lazy playlist extraction (#27829) -* [youporn] Restrict fallback download URL (#27822) -* [youporn] Improve height and tbr extraction (#20425, #23659) -* [youporn] Fix extraction (#27822) -+ [twitter] Add support for unified cards (#27826) -+ [twitch] Add Authorization header with OAuth token for GraphQL requests - (#27790) -* [mixcloud:playlist:base] Extract video id in flat playlist mode (#27787) -* [cspan] Improve info extraction (#27791) -* [adn] Improve info extraction -* [adn] Fix extraction (#26963, #27732) -* [youtube:search] Extract from all sections (#27604) -* [youtube:search] fix viewcount and try to extract all video sections (#27604) -* [twitch] Improve login error extraction -* [twitch] Fix authentication (#27743) -* [3qsdn] Improve extraction (#21058) -* [peertube] Extract formats from streamingPlaylists (#26002, #27586, #27728) -* [khanacademy] Fix extraction (#2887, #26803) -* [spike] Update Paramount Network feed URL (#27715) - - -version 2021.01.08 - -Core -* [downloader/hls] Disable decryption in tests (#27660) -+ [utils] Add a function to clean podcast URLs - -Extractors -* [rai] Improve subtitles extraction (#27698, #27705) -* [canvas] Match only supported VRT NU URLs (#27707) -+ [bibeltv] Add support for bibeltv.de (#14361) -+ [bfmtv] Add support for bfmtv.com (#16053, #26615) -+ [sbs] Add support for ondemand play and news embed URLs (#17650, #27629) -* [twitch] Drop legacy kraken API v5 code altogether and refactor -* [twitch:vod] Switch to GraphQL for video metadata -* [canvas] Fix VRT NU extraction (#26957, #27053) -* [twitch] Switch access token to GraphQL and refactor (#27646) -+ [rai] Detect ContentItem in iframe (#12652, #27673) -* [ketnet] Fix extraction (#27662) -+ [dplay] Add suport Discovery+ domains (#27680) -* [motherless] Improve extraction (#26495, #27450) -* [motherless] Fix recent videos upload date extraction (#27661) -* [nrk] Fix extraction for videos without a legalAge rating -- [googleplus] Remove extractor (#4955, #7400) -+ [applepodcasts] Add support for podcasts.apple.com (#25918) -+ [googlepodcasts] Add support for podcasts.google.com -+ [iheart] Add support for iheart.com (#27037) -* [acast] Clean podcast URLs -* [stitcher] Clean podcast URLs -+ [xfileshare] Add support for aparat.cam (#27651) -+ [twitter] Add support for summary card (#25121) -* [twitter] Try to use a Generic fallback for unknown twitter cards (#25982) -+ [stitcher] Add support for shows and show metadata extraction (#20510) -* [stv] Improve episode id extraction (#23083) - - -version 2021.01.03 - -Extractors -* [nrk] Improve series metadata extraction (#27473) -+ [nrk] Extract subtitles -* [nrk] Fix age limit extraction -* [nrk] Improve video id extraction -+ [nrk] Add support for podcasts (#27634, #27635) -* [nrk] Generalize and delegate all item extractors to nrk -+ [nrk] Add support for mp3 formats -* [nrktv] Switch to playback endpoint -* [vvvvid] Fix season metadata extraction (#18130) -* [stitcher] Fix extraction (#20811, #27606) -* [acast] Fix extraction (#21444, #27612, #27613) -+ [arcpublishing] Add support for arcpublishing.com (#2298, #9340, #17200) -+ [sky] Add support for Sports News articles and Brighcove videos (#13054) -+ [vvvvid] Extract akamai formats -* [vvvvid] Skip unplayable episodes (#27599) -* [yandexvideo] Fix extraction for Python 3.4 - - -version 2020.12.31 - -Core -* [utils] Accept only supported protocols in url_or_none -* [YoutubeDL] Allow format filtering using audio language (#16209) - -Extractors -+ [redditr] Extract all thumbnails (#27503) -* [vvvvid] Improve info extraction -+ [vvvvid] Add support for playlists (#18130, #27574) -+ [yandexdisk] Extract info from webpage -* [yandexdisk] Fix extraction (#17861, #27131) -* [yandexvideo] Use old API call as fallback -* [yandexvideo] Fix extraction (#25000) -- [nbc] Remove CSNNE extractor -* [nbc] Fix NBCSport VPlayer URL extraction (#16640) -+ [aenetworks] Add support for biography.com (#3863) -* [uktvplay] Match new video URLs (#17909) -* [sevenplay] Detect API errors -* [tenplay] Fix format extraction (#26653) -* [brightcove] Raise error for DRM protected videos (#23467, #27568) - - -version 2020.12.29 - -Extractors -* [youtube] Improve yt initial data extraction (#27524) -* [youtube:tab] Improve URL matching #27559) -* [youtube:tab] Restore retry on browse requests (#27313, #27564) -* [aparat] Fix extraction (#22285, #22611, #23348, #24354, #24591, #24904, - #25418, #26070, #26350, #26738, #27563) -- [brightcove] Remove sonyliv specific code -* [piksel] Improve format extraction -+ [zype] Add support for uplynk videos -+ [toggle] Add support for live.mewatch.sg (#27555) -+ [go] Add support for fxnow.fxnetworks.com (#13972, #22467, #23754, #26826) -* [teachable] Improve embed detection (#26923) -* [mitele] Fix free video extraction (#24624, #25827, #26757) -* [telecinco] Fix extraction -* [youtube] Update invidious.snopyta.org (#22667) -* [amcnetworks] Improve auth only video detection (#27548) -+ [generic] Add support for VHX Embeds (#27546) - - -version 2020.12.26 - -Extractors -* [instagram] Fix comment count extraction -+ [instagram] Add support for reel URLs (#26234, #26250) -* [bbc] Switch to media selector v6 (#23232, #23933, #26303, #26432, #26821, - #27538) -* [instagram] Improve thumbnail extraction -* [instagram] Fix extraction when authenticated (#22880, #26377, #26981, - #27422) -* [spankbang:playlist] Fix extraction (#24087) -+ [spankbang] Add support for playlist videos -* [pornhub] Improve like and dislike count extraction (#27356) -* [pornhub] Fix lq formats extraction (#27386, #27393) -+ [bongacams] Add support for bongacams.com (#27440) -* [youtube:tab] Extend URL regular expression (#27501) -* [theweatherchannel] Fix extraction (#25930, #26051) -+ [sprout] Add support for Universal Kids (#22518) -* [theplatform] Allow passing geo bypass countries from other extractors -+ [wistia] Add support for playlists (#27533) -+ [ctv] Add support for ctv.ca (#27525) -* [9c9media] Improve info extraction -* [youtube] Fix automatic captions extraction (#27162, #27388) -* [sonyliv] Fix title for movies -* [sonyliv] Fix extraction (#25667) -* [streetvoice] Fix extraction (#27455, #27492) -+ [facebook] Add support for watchparty pages (#27507) -* [cbslocal] Fix video extraction -+ [brightcove] Add another method to extract policyKey -* [mewatch] Relax URL regular expression (#27506) - - -version 2020.12.22 - -Core -* [common] Remove unwanted query params from unsigned akamai manifest URLs - -Extractors -- [tastytrade] Remove extractor (#25716) -* [niconico] Fix playlist extraction (#27428) -- [everyonesmixtape] Remove extractor -- [kanalplay] Remove extractor -* [arkena] Fix extraction -* [nba] Rewrite extractor -* [turner] Improve info extraction -* [youtube] Improve xsrf token extraction (#27442) -* [generic] Improve RSS age limit extraction -* [generic] Fix RSS itunes thumbnail extraction (#27405) -+ [redditr] Extract duration (#27426) -- [zaq1] Remove extractor -+ [asiancrush] Add support for retrocrush.tv -* [asiancrush] Fix extraction -- [noco] Remove extractor (#10864) -* [nfl] Fix extraction (#22245) -* [skysports] Relax URL regular expression (#27435) -+ [tv5unis] Add support for tv5unis.ca (#22399, #24890) -+ [videomore] Add support for more.tv (#27088) -+ [yandexmusic] Add support for music.yandex.com (#27425) -+ [nhk:program] Add support for audio programs and program clips -+ [nhk] Add support for NHK video programs (#27230) - - -version 2020.12.14 - -Core -* [extractor/common] Improve JSON-LD interaction statistic extraction (#23306) -* [downloader/hls] Delegate manifests with media initialization to ffmpeg -+ [extractor/common] Document duration meta field for playlists - -Extractors -* [mdr] Bypass geo restriction -* [mdr] Improve extraction (#24346, #26873) -* [yandexmusic:album] Improve album title extraction (#27418) -* [eporner] Fix view count extraction and make optional (#23306) -+ [eporner] Extend URL regular expression -* [eporner] Fix hash extraction and extend _VALID_URL (#27396) -* [slideslive] Use m3u8 entry protocol for m3u8 formats (#27400) -* [twitcasting] Fix format extraction and improve info extraction (#24868) -* [linuxacademy] Fix authentication and extraction (#21129, #26223, #27402) -* [itv] Clean description from HTML tags (#27399) -* [vlive] Sort live formats (#27404) -* [hotstart] Fix and improve extraction - * Fix format extraction (#26690) - + Extract thumbnail URL (#16079, #20412) - + Add support for country specific playlist URLs (#23496) - * Select the last id in video URL (#26412) -+ [youtube] Add some invidious instances (#27373) - - -version 2020.12.12 - -Core -* [YoutubeDL] Improve thumbnail filename deducing (#26010, #27244) - -Extractors -+ [ruutu] Extract more metadata -+ [ruutu] Detect non-free videos (#21154) -* [ruutu] Authenticate format URLs (#21031, #26782) -+ [ruutu] Add support for static.nelonenmedia.fi (#25412) -+ [ruutu] Extend URL regular expression (#24839) -+ [facebook] Add support archived live video URLs (#15859) -* [wdr] Improve overall extraction -+ [wdr] Extend subtitles extraction (#22672, #22723) -+ [facebook] Add support for videos attached to Relay based story pages - (#10795) -+ [wdr:page] Add support for kinder.wdr.de (#27350) -+ [facebook] Add another regular expression for handleServerJS -* [facebook] Fix embed page extraction -+ [facebook] Add support for Relay post pages (#26935) -+ [facebook] Add support for watch videos (#22795, #27062) -+ [facebook] Add support for group posts with multiple videos (#19131) -* [itv] Fix series metadata extraction (#26897) -- [itv] Remove old extraction method (#23177) -* [facebook] Redirect mobile URLs to desktop URLs (#24831, #25624) -+ [facebook] Add support for Relay based pages (#26823) -* [facebook] Try to reduce unnecessary tahoe requests -- [facebook] Remove hardcoded Chrome User-Agent (#18974, #25411, #26958, - #27329) -- [smotri] Remove extractor (#27358) -- [beampro] Remove extractor (#17290, #22871, #23020, #23061, #26099) - - -version 2020.12.09 - -Core -* [extractor/common] Fix inline HTML5 media tags processing (#27345) - -Extractors -* [youtube:tab] Improve identity token extraction (#27197) -* [youtube:tab] Make click tracking params on continuation optional -* [youtube:tab] Delegate inline playlists to tab-based playlists (27298) -+ [tubitv] Extract release year (#27317) -* [amcnetworks] Fix free content extraction (#20354) -+ [lbry:channel] Add support for channels (#25584) -+ [lbry] Add support for short and embed URLs -* [lbry] Fix channel metadata extraction -+ [telequebec] Add support for video.telequebec.tv (#27339) -* [telequebec] Fix extraction (#25733, #26883) -+ [youtube:tab] Capture and output alerts (#27340) -* [tvplay:home] Fix extraction (#21153) -* [americastestkitchen] Fix Extraction and add support - for Cook's Country and Cook's Illustrated (#17234, #27322) -+ [slideslive] Add support for yoda service videos and extract subtitles - (#27323) - - -version 2020.12.07 - -Core -* [extractor/common] Extract timestamp from Last-Modified header -+ [extractor/common] Add support for dl8-* media tags (#27283) -* [extractor/common] Fix media type extraction for HTML5 media tags - in start/end form - -Extractors -* [aenetworks] Fix extraction (#23363, #23390, #26795, #26985) - * Fix Fastly format extraction - + Add support for play and watch subdomains - + Extract series metadata -* [youtube] Improve youtu.be extraction in non-existing playlists (#27324) -+ [generic] Extract RSS video description, timestamp and itunes metadata - (#27177) -* [nrk] Reduce the number of instalments and episodes requests -* [nrk] Improve extraction - * Improve format extraction for old akamai formats - + Add is_live value to entry info dict - * Request instalments only when available - * Fix skole extraction -+ [peertube] Extract fps -+ [peertube] Recognize audio-only formats (#27295) - - -version 2020.12.05 - -Core -* [extractor/common] Improve Akamai HTTP format extraction - * Allow m3u8 manifest without an additional audio format - * Fix extraction for qualities starting with a number - -Extractors -* [teachable:course] Improve extraction (#24507, #27286) -* [nrk] Improve error extraction -* [nrktv:series] Improve extraction (#21926) -* [nrktv:season] Improve extraction -* [nrk] Improve format extraction and geo-restriction detection (#24221) -* [pornhub] Handle HTTP errors gracefully (#26414) -* [nrktv] Relax URL regular expression (#27299, #26185) -+ [zdf] Extract webm formats (#26659) -+ [gamespot] Extract DASH and HTTP formats -+ [tver] Add support for tver.jp (#26662, #27284) -+ [pornhub] Add support for pornhub.org (#27276) - - -version 2020.12.02 - -Extractors -+ [tva] Add support for qub.ca (#27235) -+ [toggle] Detect DRM protected videos (#16479, #20805) -+ [toggle] Add support for new MeWatch URLs (#27256) -* [youtube:tab] Extract channels only from channels tab (#27266) -+ [cspan] Extract info from jwplayer data (#3672, #3734, #10638, #13030, - #18806, #23148, #24461, #26171, #26800, #27263) -* [cspan] Pass Referer header with format's video URL (#26032, #25729) -* [youtube] Improve age-gated videos extraction (#27259) -+ [mediaset] Add support for movie URLs (#27240) -* [yandexmusic] Refactor -+ [yandexmusic] Add support for artist's tracks and albums (#11887, #22284) -* [yandexmusic:track] Fix extraction (#26449, #26669, #26747, #26748, #26762) - - -version 2020.11.29 - -Core -* [YoutubeDL] Write static debug to stderr and respect quiet for dynamic debug - (#14579, #22593) - -Extractors -* [drtv] Extend URL regular expression (#27243) -* [tiktok] Fix extraction (#20809, #22838, #22850, #25987, #26281, #26411, - #26639, #26776, #27237) -+ [ina] Add support for mobile URLs (#27229) -* [pornhub] Fix like and dislike count extraction (#27227, #27234) -* [youtube] Improve yt initial player response extraction (#27216) -* [videa] Fix extraction (#25650, #25973, #26301) - - -version 2020.11.26 - -Core -* [downloader/fragment] Set final file's mtime according to last fragment's - Last-Modified header (#11718, #18384, #27138) - -Extractors -+ [spreaker] Add support for spreaker.com (#13480, #13877) -* [vlive] Improve extraction for geo-restricted videos -+ [vlive] Add support for post URLs (#27122, #27123) -* [viki] Fix video API request (#27184) -* [bbc] Fix BBC Three clip extraction -* [bbc] Fix BBC News videos extraction -+ [medaltv] Add support for medal.tv (#27149) -* [youtube] Improve music metadata and license extraction (#26013) -* [nrk] Fix extraction -* [cda] Fix extraction (#17803, #24458, #24518, #26381) - - -version 2020.11.24 - -Core -+ [extractor/common] Add generic support for akamai HTTP format extraction - -Extractors -* [youtube:tab] Fix feeds extraction (#25695, #26452) -* [youtube:favorites] Restore extractor -* [youtube:tab] Fix some weird typo (#27157) -+ [pinterest] Add support for large collections (more than 25 pins) -+ [franceinter] Extract thumbnail (#27153) -+ [box] Add support for box.com (#5949) -+ [nytimes] Add support for cooking.nytimes.com (#27112, #27143) -* [lbry] Relax URL regular expression (#27144) -+ [rumble] Add support for embed pages (#10785) -+ [skyit] Add support for multiple Sky Italia websites (#26629) -+ [pinterest] Add support for pinterest.com (#25747) - - -version 2020.11.21.1 - -Core -* [downloader/http] Fix crash during urlopen caused by missing reason - of URLError -* [YoutubeDL] Fix --ignore-errors for playlists with generator-based entries - of url_transparent (#27064) - -Extractors -+ [svtplay] Add support for svt.se/barnkanalen (#24817) -+ [svt] Extract timestamp (#27130) -* [svtplay] Improve thumbnail extraction (#27130) -* [youtube] Fix error reason extraction (#27081) -* [youtube] Fix like and dislike count extraction (#25977) -+ [youtube:tab] Add support for current video and fix lives extraction (#27126) -* [infoq] Fix format extraction (#25984) -* [francetv] Update to fix thumbnail URL issue (#27120) -* [youtube] Improve yt initial data extraction (#27093) -+ [discoverynetworks] Add support new TLC/DMAX URLs (#27100) -* [rai] Fix protocol relative relinker URLs (#22766) -* [rai] Fix unavailable video format detection -* [rai] Improve extraction -* [rai] Fix extraction (#27077) -* [viki] Improve format extraction -* [viki] Fix stream extraction from MPD (#27092) -* [googledrive] Fix format extraction (#26979) -+ [amara] Add support for amara.org (#20618) -* [vimeo:album] Fix extraction (#27079) -* [mtv] Fix mgid extraction (#26841) - - -version 2020.11.19 - -Core -* [extractor/common] Output error for invalid URLs in _is_valid_url (#21400, - #24151, #25617, #25618, #25586, #26068, #27072) - -Extractors -* [youporn] Fix upload date extraction -* [youporn] Make comment count optional (#26986) -* [arte] Rework extractors - * Reimplement embed and playlist extractors to delegate to the single - entrypoint artetv extractor - * Improve embeds detection (#27057) -+ [arte] Extract m3u8 formats (#27061) -* [mgtv] Fix format extraction (#26415) -+ [lbry] Add support for odysee.com (#26806) -* [francetv] Improve info extraction -+ [francetv] Add fallback video URL extraction (#27047) - - -version 2020.11.18 - -Extractors -* [spiegel] Fix extraction (#24206, #24767) -* [youtube] Improve extraction - + Add support for --no-playlist (#27009) - * Improve playlist and mix extraction (#26390, #26509, #26534, #27011) - + Extract playlist uploader data -* [youtube:tab] Fix view count extraction (#27051) -* [malltv] Fix extraction (#27035) -+ [bandcamp] Extract playlist description (#22684) -* [urplay] Fix extraction (#26828) -* [youtube:tab] Fix playlist title extraction (#27015) -* [youtube] Fix chapters extraction (#26005) - - -version 2020.11.17 - -Core -* [utils] Skip ! prefixed code in js_to_json - -Extractors -* [youtube:tab] Fix extraction with cookies provided (#27005) -* [lrt] Fix extraction with empty tags (#20264) -+ [ndr:embed:base] Extract subtitles (#25447, #26106) -+ [servus] Add support for pm-wissen.com (#25869) -* [servus] Fix extraction (#26872, #26967, #26983, #27000) -* [xtube] Fix extraction (#26996) -* [lrt] Fix extraction -+ [lbry] Add support for lbry.tv -+ [condenast] Extract subtitles -* [condenast] Fix extraction -* [bandcamp] Fix extraction (#26681, #26684) -* [rai] Fix RaiPlay extraction (#26064, #26096) -* [vlive] Fix extraction -* [usanetwork] Fix extraction -* [nbc] Fix NBCNews/Today/MSNBC extraction -* [cnbc] Fix extraction - - -version 2020.11.12 - -Extractors -* [youtube] Rework extractors - - -version 2020.11.01 - -Core -* [utils] Don't attempt to coerce JS strings to numbers in js_to_json (#26851) -* [downloader/http] Properly handle missing message in SSLError (#26646) -* [downloader/http] Fix access to not yet opened stream in retry - -Extractors -* [youtube] Fix JS player URL extraction -* [ytsearch] Fix extraction (#26920) -* [afreecatv] Fix typo (#26970) -* [23video] Relax URL regular expression (#26870) -+ [ustream] Add support for video.ibm.com (#26894) -* [iqiyi] Fix typo (#26884) -+ [expressen] Add support for di.se (#26670) -* [iprima] Improve video id extraction (#26507, #26494) - - version 2020.09.20 Core @@ -1156,7 +283,7 @@ Extractors + Add support for more domains * [svt] Fix series extraction (#22297) * [svt] Fix article extraction (#22897, #22919) -* [soundcloud] Improve private playlist/set tracks extraction (#3707) +* [soundcloud] Imporve private playlist/set tracks extraction (#3707) version 2020.01.24 @@ -1282,7 +409,7 @@ Extractors * [abcotvs] Relax URL regular expression and improve metadata extraction (#18014) * [channel9] Reduce response size -* [adobetv] Improve extraction +* [adobetv] Improve extaction * Use OnDemandPagedList for list extractors * Reduce show extraction requests * Extract original video format and subtitles @@ -1307,7 +434,7 @@ Extractors * [dailymotion] Improve extraction * Extract http formats included in m3u8 manifest * Fix user extraction (#3553, #21415) - + Add support for User Authentication (#11491) + + Add suport for User Authentication (#11491) * Fix password protected videos extraction (#23176) * Respect age limit option and family filter cookie value (#18437) * Handle video url playlist query param @@ -1392,7 +519,7 @@ Extractors - [go90] Remove extractor * [kakao] Remove raw request + [kakao] Extract format total bitrate -* [daum] Fix VOD and Clip extraction (#15015) +* [daum] Fix VOD and Clip extracton (#15015) * [kakao] Improve extraction + Add support for embed URLs + Add support for Kakao Legacy vid based embed URLs @@ -1436,7 +563,7 @@ Extractors * Improve format extraction (#22123) + Extract uploader_id and uploader_url (#21916) + Extract all known thumbnails (#19071, #20659) - * Fix extraction for private playlists (#20976) + * Fix extration for private playlists (#20976) + Add support for playlist embeds (#20976) * Skip preview formats (#22806) * [dplay] Improve extraction @@ -1911,7 +1038,7 @@ Extractors * [hbo] Fix extraction and extract subtitles (#14629, #13709) * [youtube] Extract srv[1-3] subtitle formats (#20566) * [adultswim] Fix extraction (#18025) -* [teamcoco] Fix extraction and add support for subdomains (#17099, #20339) +* [teamcoco] Fix extraction and add suport for subdomains (#17099, #20339) * [adn] Fix subtitle compatibility with ffmpeg * [adn] Fix extraction and add support for positioning styles (#20549) * [vk] Use unique video id (#17848) @@ -2323,7 +1450,7 @@ version 2018.11.18 Extractors + [wwe] Extract subtitles -+ [wwe] Add support for playlists (#14781) ++ [wwe] Add support for playlistst (#14781) + [wwe] Add support for wwe.com (#14781, #17450) * [vk] Detect geo restriction (#17767) * [openload] Use original host during extraction (#18211) @@ -3356,7 +2483,7 @@ Extractors * [youku] Update ccode (#14872) * [mnet] Fix format extraction (#14883) + [xiami] Add Referer header to API request -* [mtv] Correct scc extension in extracted subtitles (#13730) +* [mtv] Correct scc extention in extracted subtitles (#13730) * [vvvvid] Fix extraction for kenc videos (#13406) + [br] Add support for BR Mediathek videos (#14560, #14788) + [daisuki] Add support for motto.daisuki.com (#14681) @@ -3377,7 +2504,7 @@ Extractors * [nexx] Extract more formats + [openload] Add support for openload.link (#14763) * [empflix] Relax URL regular expression -* [empflix] Fix extraction +* [empflix] Fix extractrion * [tnaflix] Don't modify download URLs (#14811) - [gamersyde] Remove extractor * [francetv:generationwhat] Fix extraction @@ -3572,7 +2699,7 @@ Extractors * [yahoo] Bypass geo restriction for brightcove (#14210) * [yahoo] Use extracted brightcove account id (#14210) * [rtve:alacarta] Fix extraction (#14290) -+ [yahoo] Add support for custom brightcove embeds (#14210) ++ [yahoo] Add support for custom brigthcove embeds (#14210) + [generic] Add support for Video.js embeds + [gfycat] Add support for /gifs/detail URLs (#14322) * [generic] Fix infinite recursion for twitter:player URLs (#14339) @@ -3817,7 +2944,7 @@ Extractors * [amcnetworks] Make rating optional (#12453) * [cloudy] Fix extraction (#13737) + [nickru] Add support for nickelodeon.ru -* [mtv] Improve thumbnail extraction +* [mtv] Improve thumbnal extraction * [nick] Automate geo-restriction bypass (#13711) * [niconico] Improve error reporting (#13696) @@ -4181,7 +3308,7 @@ Extractors + [cda] Support birthday verification (#12789) * [leeco] Fix extraction (#12974) + [pbs] Extract chapters -* [amp] Improve thumbnail and subtitles extraction +* [amp] Imporove thumbnail and subtitles extraction * [foxsports] Fix extraction (#12945) - [coub] Remove comment count extraction (#12941) @@ -4351,7 +3478,7 @@ Extractors + [rbmaradio] Add support for redbullradio.com URLs (#12687) + [npo:live] Add support for default URL (#12555) * [mixcloud:playlist] Fix title, description and view count extraction (#12582) -+ [thesun] Add support for thesun.co.uk (#11298, #12674) ++ [thesun] Add suport for thesun.co.uk (#11298, #12674) + [ceskateleveize:porady] Add support for porady (#7411, #12645) * [ceskateleveize] Improve extraction and remove URL replacement hacks + [kaltura] Add support for iframe embeds (#12679) @@ -4390,7 +3517,7 @@ Extractors * [funimation] Fix extraction (#10696, #11773) + [xfileshare] Add support for vidabc.com (#12589) + [xfileshare] Improve extraction and extract hls formats -+ [crunchyroll] Pass geo verification proxy ++ [crunchyroll] Pass geo verifcation proxy + [cwtv] Extract ISM formats + [tvplay] Bypass geo restriction + [vrv] Add support for vrv.co @@ -4454,7 +3581,7 @@ Extractors + [bostonglobe] Add extractor for bostonglobe.com (#12099) + [toongoggles] Add support for toongoggles.com (#12171) + [medialaan] Add support for Medialaan sites (#9974, #11912) -+ [discoverynetworks] Add support for more domains and bypass geo restriction ++ [discoverynetworks] Add support for more domains and bypass geo restiction * [openload] Fix extraction (#10408) @@ -6044,7 +5171,7 @@ version 2016.07.09.1 Fixed/improved extractors - youtube - ard -- srmediathek (#9373) +- srmediatek (#9373) version 2016.07.09 @@ -6108,7 +5235,7 @@ Fixed/improved extractors - kaltura (#5557) - la7 - Changed features -- Rename --cn-verification-proxy to --geo-verification-proxy +- Rename --cn-verfication-proxy to --geo-verification-proxy Miscellaneous - Add script for displaying downloads statistics diff --git a/README.md b/README.md index 47e686f84..45326c69e 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,4 @@ -[![Build Status](https://github.com/ytdl-org/youtube-dl/workflows/CI/badge.svg)](https://github.com/ytdl-org/youtube-dl/actions?query=workflow%3ACI) - +[![Build Status](https://travis-ci.org/ytdl-org/youtube-dl.svg?branch=master)](https://travis-ci.org/ytdl-org/youtube-dl) youtube-dl - download videos from youtube.com or other video platforms @@ -33,7 +32,7 @@ Windows users can [download an .exe file](https://yt-dl.org/latest/youtube-dl.ex You can also use pip: sudo -H pip install --upgrade youtube-dl - + This command will update youtube-dl if you have already installed it. See the [pypi page](https://pypi.python.org/pypi/youtube_dl) for more information. macOS users can install youtube-dl with [Homebrew](https://brew.sh/): @@ -52,431 +51,394 @@ Alternatively, refer to the [developer instructions](#developer-instructions) fo youtube-dl [OPTIONS] URL [URL...] # OPTIONS - -h, --help Print this help text and exit - --version Print program version and exit - -U, --update Update this program to latest version. - Make sure that you have sufficient - permissions (run with sudo if needed) - -i, --ignore-errors Continue on download errors, for - example to skip unavailable videos in a - playlist - --abort-on-error Abort downloading of further videos (in - the playlist or the command line) if an - error occurs - --dump-user-agent Display the current browser - identification - --list-extractors List all supported extractors - --extractor-descriptions Output descriptions of all supported - extractors - --force-generic-extractor Force extraction to use the generic - extractor - --default-search PREFIX Use this prefix for unqualified URLs. - For example "gvsearch2:" downloads two - videos from google videos for youtube- - dl "large apple". Use the value "auto" - to let youtube-dl guess ("auto_warning" - to emit a warning when guessing). - "error" just throws an error. The - default value "fixup_error" repairs - broken URLs, but emits an error if this - is not possible instead of searching. - --ignore-config Do not read configuration files. When - given in the global configuration file - /etc/youtube-dl.conf: Do not read the - user configuration in - ~/.config/youtube-dl/config - (%APPDATA%/youtube-dl/config.txt on - Windows) - --config-location PATH Location of the configuration file; - either the path to the config or its - containing directory. - --flat-playlist Do not extract the videos of a - playlist, only list them. - --mark-watched Mark videos watched (YouTube only) - --no-mark-watched Do not mark videos watched (YouTube - only) - --no-color Do not emit color codes in output + -h, --help Print this help text and exit + --version Print program version and exit + -U, --update Update this program to latest version. Make + sure that you have sufficient permissions + (run with sudo if needed) + -i, --ignore-errors Continue on download errors, for example to + skip unavailable videos in a playlist + --abort-on-error Abort downloading of further videos (in the + playlist or the command line) if an error + occurs + --dump-user-agent Display the current browser identification + --list-extractors List all supported extractors + --extractor-descriptions Output descriptions of all supported + extractors + --force-generic-extractor Force extraction to use the generic + extractor + --default-search PREFIX Use this prefix for unqualified URLs. For + example "gvsearch2:" downloads two videos + from google videos for youtube-dl "large + apple". Use the value "auto" to let + youtube-dl guess ("auto_warning" to emit a + warning when guessing). "error" just throws + an error. The default value "fixup_error" + repairs broken URLs, but emits an error if + this is not possible instead of searching. + --ignore-config Do not read configuration files. When given + in the global configuration file + /etc/youtube-dl.conf: Do not read the user + configuration in ~/.config/youtube- + dl/config (%APPDATA%/youtube-dl/config.txt + on Windows) + --config-location PATH Location of the configuration file; either + the path to the config or its containing + directory. + --flat-playlist Do not extract the videos of a playlist, + only list them. + --mark-watched Mark videos watched (YouTube only) + --no-mark-watched Do not mark videos watched (YouTube only) + --no-color Do not emit color codes in output ## Network Options: - --proxy URL Use the specified HTTP/HTTPS/SOCKS - proxy. To enable SOCKS proxy, specify a - proper scheme. For example - socks5://127.0.0.1:1080/. Pass in an - empty string (--proxy "") for direct - connection - --socket-timeout SECONDS Time to wait before giving up, in - seconds - --source-address IP Client-side IP address to bind to - -4, --force-ipv4 Make all connections via IPv4 - -6, --force-ipv6 Make all connections via IPv6 + --proxy URL Use the specified HTTP/HTTPS/SOCKS proxy. + To enable SOCKS proxy, specify a proper + scheme. For example + socks5://127.0.0.1:1080/. Pass in an empty + string (--proxy "") for direct connection + --socket-timeout SECONDS Time to wait before giving up, in seconds + --source-address IP Client-side IP address to bind to + -4, --force-ipv4 Make all connections via IPv4 + -6, --force-ipv6 Make all connections via IPv6 ## Geo Restriction: - --geo-verification-proxy URL Use this proxy to verify the IP address - for some geo-restricted sites. The - default proxy specified by --proxy (or - none, if the option is not present) is - used for the actual downloading. - --geo-bypass Bypass geographic restriction via - faking X-Forwarded-For HTTP header - --no-geo-bypass Do not bypass geographic restriction - via faking X-Forwarded-For HTTP header - --geo-bypass-country CODE Force bypass geographic restriction - with explicitly provided two-letter ISO - 3166-2 country code - --geo-bypass-ip-block IP_BLOCK Force bypass geographic restriction - with explicitly provided IP block in - CIDR notation + --geo-verification-proxy URL Use this proxy to verify the IP address for + some geo-restricted sites. The default + proxy specified by --proxy (or none, if the + option is not present) is used for the + actual downloading. + --geo-bypass Bypass geographic restriction via faking + X-Forwarded-For HTTP header + --no-geo-bypass Do not bypass geographic restriction via + faking X-Forwarded-For HTTP header + --geo-bypass-country CODE Force bypass geographic restriction with + explicitly provided two-letter ISO 3166-2 + country code + --geo-bypass-ip-block IP_BLOCK Force bypass geographic restriction with + explicitly provided IP block in CIDR + notation ## Video Selection: - --playlist-start NUMBER Playlist video to start at (default is - 1) - --playlist-end NUMBER Playlist video to end at (default is - last) - --playlist-items ITEM_SPEC Playlist video items to download. - Specify indices of the videos in the - playlist separated by commas like: "-- - playlist-items 1,2,5,8" if you want to - download videos indexed 1, 2, 5, 8 in - the playlist. You can specify range: " - --playlist-items 1-3,7,10-13", it will - download the videos at index 1, 2, 3, - 7, 10, 11, 12 and 13. - --match-title REGEX Download only matching titles (regex or - caseless sub-string) - --reject-title REGEX Skip download for matching titles - (regex or caseless sub-string) - --max-downloads NUMBER Abort after downloading NUMBER files - --min-filesize SIZE Do not download any videos smaller than - SIZE (e.g. 50k or 44.6m) - --max-filesize SIZE Do not download any videos larger than - SIZE (e.g. 50k or 44.6m) - --date DATE Download only videos uploaded in this - date - --datebefore DATE Download only videos uploaded on or - before this date (i.e. inclusive) - --dateafter DATE Download only videos uploaded on or - after this date (i.e. inclusive) - --min-views COUNT Do not download any videos with less - than COUNT views - --max-views COUNT Do not download any videos with more - than COUNT views - --match-filter FILTER Generic video filter. Specify any key - (see the "OUTPUT TEMPLATE" for a list - of available keys) to match if the key - is present, !key to check if the key is - not present, key > NUMBER (like - "comment_count > 12", also works with - >=, <, <=, !=, =) to compare against a - number, key = 'LITERAL' (like "uploader - = 'Mike Smith'", also works with !=) to - match against a string literal and & to - require multiple matches. Values which - are not known are excluded unless you - put a question mark (?) after the - operator. For example, to only match - videos that have been liked more than - 100 times and disliked less than 50 - times (or the dislike functionality is - not available at the given service), - but who also have a description, use - --match-filter "like_count > 100 & - dislike_count NUMBER (like "comment_count + > 12", also works with >=, <, <=, !=, =) to + compare against a number, key = 'LITERAL' + (like "uploader = 'Mike Smith'", also works + with !=) to match against a string literal + and & to require multiple matches. Values + which are not known are excluded unless you + put a question mark (?) after the operator. + For example, to only match videos that have + been liked more than 100 times and disliked + less than 50 times (or the dislike + functionality is not available at the given + service), but who also have a description, + use --match-filter "like_count > 100 & + dislike_count .+?) - (?P.+)" - --xattrs Write metadata to the video file's - xattrs (using dublin core and xdg - standards) - --fixup POLICY Automatically correct known faults of - the file. One of never (do nothing), - warn (only emit a warning), - detect_or_warn (the default; fix file - if we can, warn otherwise) - --prefer-avconv Prefer avconv over ffmpeg for running - the postprocessors - --prefer-ffmpeg Prefer ffmpeg over avconv for running - the postprocessors (default) - --ffmpeg-location PATH Location of the ffmpeg/avconv binary; - either the path to the binary or its - containing directory. - --exec CMD Execute a command on the file after - downloading and post-processing, - similar to find's -exec syntax. - Example: --exec 'adb push {} - /sdcard/Music/ && rm {}' - --convert-subs FORMAT Convert the subtitles to other format - (currently supported: srt|ass|vtt|lrc) + -x, --extract-audio Convert video files to audio-only files + (requires ffmpeg or avconv and ffprobe or + avprobe) + --audio-format FORMAT Specify audio format: "best", "aac", + "flac", "mp3", "m4a", "opus", "vorbis", or + "wav"; "best" by default; No effect without + -x + --audio-quality QUALITY Specify ffmpeg/avconv audio quality, insert + a value between 0 (better) and 9 (worse) + for VBR or a specific bitrate like 128K + (default 5) + --recode-video FORMAT Encode the video to another format if + necessary (currently supported: + mp4|flv|ogg|webm|mkv|avi) + --postprocessor-args ARGS Give these arguments to the postprocessor + -k, --keep-video Keep the video file on disk after the post- + processing; the video is erased by default + --no-post-overwrites Do not overwrite post-processed files; the + post-processed files are overwritten by + default + --embed-subs Embed subtitles in the video (only for mp4, + webm and mkv videos) + --embed-thumbnail Embed thumbnail in the audio as cover art + --add-metadata Write metadata to the video file + --metadata-from-title FORMAT Parse additional metadata like song title / + artist from the video title. The format + syntax is the same as --output. Regular + expression with named capture groups may + also be used. The parsed parameters replace + existing values. Example: --metadata-from- + title "%(artist)s - %(title)s" matches a + title like "Coldplay - Paradise". Example + (regex): --metadata-from-title + "(?P<artist>.+?) - (?P<title>.+)" + --xattrs Write metadata to the video file's xattrs + (using dublin core and xdg standards) + --fixup POLICY Automatically correct known faults of the + file. One of never (do nothing), warn (only + emit a warning), detect_or_warn (the + default; fix file if we can, warn + otherwise) + --prefer-avconv Prefer avconv over ffmpeg for running the + postprocessors + --prefer-ffmpeg Prefer ffmpeg over avconv for running the + postprocessors (default) + --ffmpeg-location PATH Location of the ffmpeg/avconv binary; + either the path to the binary or its + containing directory. + --exec CMD Execute a command on the file after + downloading and post-processing, similar to + find's -exec syntax. Example: --exec 'adb + push {} /sdcard/Music/ && rm {}' + --convert-subs FORMAT Convert the subtitles to other format + (currently supported: srt|ass|vtt|lrc) # CONFIGURATION @@ -563,7 +525,7 @@ The basic usage is not to set any template arguments when downloading a single f - `is_live` (boolean): Whether this video is a live stream or a fixed-length video - `start_time` (numeric): Time in seconds where the reproduction should start, as specified in the URL - `end_time` (numeric): Time in seconds where the reproduction should end, as specified in the URL - - `format` (string): A human-readable description of the format + - `format` (string): A human-readable description of the format - `format_id` (string): Format code specified by `--format` - `format_note` (string): Additional info about the format - `width` (numeric): Width of the video @@ -583,7 +545,7 @@ The basic usage is not to set any template arguments when downloading a single f - `extractor` (string): Name of the extractor - `extractor_key` (string): Key name of the extractor - `epoch` (numeric): Unix epoch when creating the file - - `autonumber` (numeric): Number that will be increased with each download, starting at `--autonumber-start` + - `autonumber` (numeric): Five-digit number that will be increased with each download, starting at zero - `playlist` (string): Name or id of the playlist that contains the video - `playlist_index` (numeric): Index of the video in the playlist padded with leading zeros according to the total length of the playlist - `playlist_id` (string): Playlist identifier @@ -620,7 +582,7 @@ Available for the media that is a track or a part of a music album: - `disc_number` (numeric): Number of the disc or other physical medium the track belongs to - `release_year` (numeric): Year (YYYY) when the album was released -Each aforementioned sequence when referenced in an output template will be replaced by the actual value corresponding to the sequence name. Note that some of the sequences are not guaranteed to be present since they depend on the metadata obtained by a particular extractor. Such sequences will be replaced with placeholder value provided with `--output-na-placeholder` (`NA` by default). +Each aforementioned sequence when referenced in an output template will be replaced by the actual value corresponding to the sequence name. Note that some of the sequences are not guaranteed to be present since they depend on the metadata obtained by a particular extractor. Such sequences will be replaced with `NA`. For example for `-o %(title)s-%(id)s.%(ext)s` and an mp4 video with title `youtube-dl test video` and id `BaW_jenozKcj`, this will result in a `youtube-dl test video-BaW_jenozKcj.mp4` file created in the current directory. @@ -632,7 +594,7 @@ To use percent literals in an output template use `%%`. To output to stdout use The current default template is `%(title)s-%(id)s.%(ext)s`. -In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title. +In some cases, you don't want special characters such as 中, spaces, or &, such as when transferring the downloaded filename to a Windows system or the filename through an 8bit-unsafe channel. In these cases, add the `--restrict-filenames` flag to get a shorter title: #### Output template and Windows batch files @@ -675,7 +637,7 @@ The general syntax for format selection is `--format FORMAT` or shorter `-f FORM **tl;dr:** [navigate me to examples](#format-selection-examples). -The simplest case is requesting a specific format, for example with `-f 22` you can download the format with format code equal to 22. You can get the list of available format codes for particular video using `--list-formats` or `-F`. Note that these format codes are extractor specific. +The simplest case is requesting a specific format, for example with `-f 22` you can download the format with format code equal to 22. You can get the list of available format codes for particular video using `--list-formats` or `-F`. Note that these format codes are extractor specific. You can also use a file extension (currently `3gp`, `aac`, `flv`, `m4a`, `mp3`, `mp4`, `ogg`, `wav`, `webm` are supported) to download the best quality format of a particular file extension served as a single file, e.g. `-f webm` will download the best quality format with the `webm` extension served as a single file. @@ -715,7 +677,6 @@ Also filtering work for comparisons `=` (equals), `^=` (starts with), `$=` (ends - `container`: Name of the container format - `protocol`: The protocol that will be used for the actual download, lower-case (`http`, `https`, `rtsp`, `rtmp`, `rtmpe`, `mms`, `f4m`, `ism`, `http_dash_segments`, `m3u8`, or `m3u8_native`) - `format_id`: A short description of the format - - `language`: Language code Any string comparison may be prefixed with negation `!` in order to produce an opposite comparison, e.g. `!*=` (does not contain). @@ -760,7 +721,7 @@ Videos can be filtered by their upload date using the options `--date`, `--dateb - Absolute dates: Dates in the format `YYYYMMDD`. - Relative dates: Dates in the format `(now|today)[+-][0-9](day|week|month|year)(s)?` - + Examples: ```bash @@ -893,7 +854,7 @@ Since June 2012 ([#342](https://github.com/ytdl-org/youtube-dl/issues/342)) yout ### The exe throws an error due to missing `MSVCR100.dll` -To run the exe you need to install first the [Microsoft Visual C++ 2010 Service Pack 1 Redistributable Package (x86)](https://download.microsoft.com/download/1/6/5/165255E7-1014-4D0A-B094-B6A430A6BFFC/vcredist_x86.exe). +To run the exe you need to install first the [Microsoft Visual C++ 2010 Redistributable Package (x86)](https://www.microsoft.com/en-US/download/details.aspx?id=5555). ### On Windows, how should I set up ffmpeg and youtube-dl? Where should I put the exe files? @@ -918,7 +879,7 @@ Either prepend `https://www.youtube.com/watch?v=` or separate the ID from the op Use the `--cookies` option, for example `--cookies /path/to/cookies/file.txt`. -In order to extract cookies from browser use any conforming browser extension for exporting cookies. For example, [Get cookies.txt LOCALLY](https://chrome.google.com/webstore/detail/get-cookiestxt-locally/cclelndahbckbenkjhflpdbgdldlbecc) (for Chrome) or [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) (for Firefox). +In order to extract cookies from browser use any conforming browser extension for exporting cookies. For example, [cookies.txt](https://chrome.google.com/webstore/detail/cookiestxt/njabckikapfpffapmjgojcnbfjonfjfg) (for Chrome) or [cookies.txt](https://addons.mozilla.org/en-US/firefox/addon/cookies-txt/) (for Firefox). Note that the cookies file must be in Mozilla/Netscape format and the first line of the cookies file must be either `# HTTP Cookie File` or `# Netscape HTTP Cookie File`. Make sure you have correct [newline format](https://en.wikipedia.org/wiki/Newline) in the cookies file and convert newlines if necessary to correspond with your OS, namely `CRLF` (`\r\n`) for Windows and `LF` (`\n`) for Unix and Unix-like systems (Linux, macOS, etc.). `HTTP Error 400: Bad Request` when using `--cookies` is a good sign of invalid newline format. @@ -1000,8 +961,6 @@ To run the test, simply invoke your favorite test runner, or execute a test file python test/test_download.py nosetests -For Python versions 3.6 and later, you can use [pynose](https://pypi.org/project/pynose/) to implement `nosetests`. The original [nose](https://pypi.org/project/nose/) has not been upgraded for 3.10 and later. - See item 6 of [new extractor tutorial](#adding-support-for-a-new-site) for how to run extractor specific test cases. If you want to create a build of youtube-dl yourself, you'll need @@ -1071,11 +1030,9 @@ After you have ensured this site is distributing its content legally, you can fo } ``` 5. Add an import in [`youtube_dl/extractor/extractors.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/extractors.py). -6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test (actually, test case) then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note: - * the test names use the extractor class name **without the trailing `IE`** - * tests with `only_matching` key in test's dict are not counted. -8. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want. -9. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart): +6. Run `python test/test_download.py TestDownload.test_YourExtractor`. This *should fail* at first, but you can continually re-run it until you're done. If you decide to add more than one test, then rename ``_TEST`` to ``_TESTS`` and make it into a list of dictionaries. The tests will then be named `TestDownload.test_YourExtractor`, `TestDownload.test_YourExtractor_1`, `TestDownload.test_YourExtractor_2`, etc. Note that tests with `only_matching` key in test's dict are not counted in. +7. Have a look at [`youtube_dl/extractor/common.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/extractor/common.py) for possible helper methods and a [detailed description of what your extractor should and may return](https://github.com/ytdl-org/youtube-dl/blob/7f41a598b3fba1bcab2817de64a08941200aa3c8/youtube_dl/extractor/common.py#L94-L303). Add tests and code for as many as you want. +8. Make sure your code follows [youtube-dl coding conventions](#youtube-dl-coding-conventions) and check the code with [flake8](https://flake8.pycqa.org/en/latest/index.html#quickstart): $ flake8 youtube_dl/extractor/yourextractor.py @@ -1093,7 +1050,7 @@ In any case, thank you very much for your contributions! ## youtube-dl coding conventions -This section introduces guidelines for writing idiomatic, robust and future-proof extractor code. +This section introduces a guide lines for writing idiomatic, robust and future-proof extractor code. Extractors are very fragile by nature since they depend on the layout of the source data provided by 3rd party media hosters out of your control and this layout tends to change. As an extractor implementer your task is not only to write code that will extract media links and metadata correctly but also to minimize dependency on the source's layout and even to make the code foresee potential future changes and be ready for that. This is important because it will allow the extractor not to break on minor layout changes thus keeping old youtube-dl versions working. Even though this breakage issue is easily fixed by emitting a new version of youtube-dl with a fix incorporated, all the previous versions become broken in all repositories and distros' packages that may not be so prompt in fetching the update from us. Needless to say, some non rolling release distros may never receive an update at all. @@ -1116,7 +1073,7 @@ Say you have some source dictionary `meta` that you've fetched as JSON with HTTP ```python meta = self._download_json(url, video_id) ``` - + Assume at this point `meta`'s layout is: ```python @@ -1160,7 +1117,7 @@ description = self._search_regex( ``` On failure this code will silently continue the extraction with `description` set to `None`. That is useful for metafields that may or may not be present. - + ### Provide fallbacks When extracting metadata try to do so from multiple sources. For example if `title` is present in several places, try extracting from at least some of them. This makes it more future-proof in case some of the sources become unavailable. @@ -1208,7 +1165,7 @@ r'(id|ID)=(?P<id>\d+)' #### Make regular expressions relaxed and flexible When using regular expressions try to write them fuzzy, relaxed and flexible, skipping insignificant parts that are more likely to change, allowing both single and double quotes for quoted values and so on. - + ##### Example Say you need to extract `title` from the following HTML code: @@ -1232,7 +1189,7 @@ title = self._search_regex( webpage, 'title', group='title') ``` -Note how you tolerate potential changes in the `style` attribute's value or switch from using double quotes to single for `class` attribute: +Note how you tolerate potential changes in the `style` attribute's value or switch from using double quotes to single for `class` attribute: The code definitely should not look like: @@ -1333,114 +1290,27 @@ Wrap all extracted numeric data into safe functions from [`youtube_dl/utils.py`] Use `url_or_none` for safe URL processing. -Use `traverse_obj` for safe metadata extraction from parsed JSON. +Use `try_get` for safe metadata extraction from parsed JSON. -Use `unified_strdate` for uniform `upload_date` or any `YYYYMMDD` meta field extraction, `unified_timestamp` for uniform `timestamp` extraction, `parse_filesize` for `filesize` extraction, `parse_count` for count meta fields extraction, `parse_resolution`, `parse_duration` for `duration` extraction, `parse_age_limit` for `age_limit` extraction. +Use `unified_strdate` for uniform `upload_date` or any `YYYYMMDD` meta field extraction, `unified_timestamp` for uniform `timestamp` extraction, `parse_filesize` for `filesize` extraction, `parse_count` for count meta fields extraction, `parse_resolution`, `parse_duration` for `duration` extraction, `parse_age_limit` for `age_limit` extraction. Explore [`youtube_dl/utils.py`](https://github.com/ytdl-org/youtube-dl/blob/master/youtube_dl/utils.py) for more useful convenience functions. #### More examples ##### Safely extract optional description from parsed JSON - -When processing complex JSON, as often returned by site API requests or stashed in web pages for "hydration", you can use the `traverse_obj()` utility function to handle multiple fallback values and to ensure the expected type of metadata items. The function's docstring defines how the function works: also review usage in the codebase for more examples. - -In this example, a text `description`, or `None`, is pulled from the `.result.video[0].summary` member of the parsed JSON `response`, if available. - -```python -description = traverse_obj(response, ('result', 'video', 0, 'summary', T(compat_str))) -``` -`T(...)` is a shorthand for a set literal; if you hate people who still run Python 2.6, `T(type_or_transformation)` could be written as a set literal `{type_or_transformation}`. - -Some extractors use the older and less capable `try_get()` function in the same way. - ```python description = try_get(response, lambda x: x['result']['video'][0]['summary'], compat_str) ``` ##### Safely extract more optional metadata - -In this example, various optional metadata values are extracted from the `.result.video[0]` member of the parsed JSON `response`, which is expected to be a JS object, parsed into a `dict`, with no crash if that isn't so, or if any of the target values are missing or invalid. - ```python -video = traverse_obj(response, ('result', 'video', 0, T(dict))) or {} -# formerly: -# video = try_get(response, lambda x: x['result']['video'][0], dict) or {} +video = try_get(response, lambda x: x['result']['video'][0], dict) or {} description = video.get('summary') duration = float_or_none(video.get('durationMs'), scale=1000) view_count = int_or_none(video.get('views')) ``` -#### Safely extract nested lists - -Suppose you've extracted JSON like this into a Python data structure named `media_json` using, say, the `_download_json()` or `_parse_json()` methods of `InfoExtractor`: -```json -{ - "title": "Example video", - "comment": "try extracting this", - "media": [{ - "type": "bad", - "size": 320, - "url": "https://some.cdn.site/bad.mp4" - }, { - "type": "streaming", - "url": "https://some.cdn.site/hls.m3u8" - }, { - "type": "super", - "size": 1280, - "url": "https://some.cdn.site/good.webm" - }], - "moreStuff": "more values", - ... -} -``` - -Then extractor code like this can collect the various fields of the JSON: -```python -... -from ..utils import ( - determine_ext, - int_or_none, - T, - traverse_obj, - txt_or_none, - url_or_none, -) -... - ... - info_dict = {} - # extract title and description if valid and not empty - info_dict.update(traverse_obj(media_json, { - 'title': ('title', T(txt_or_none)), - 'description': ('comment', T(txt_or_none)), - })) - - # extract any recognisable media formats - fmts = [] - # traverse into "media" list, extract `dict`s with desired keys - for fmt in traverse_obj(media_json, ('media', Ellipsis, { - 'format_id': ('type', T(txt_or_none)), - 'url': ('url', T(url_or_none)), - 'width': ('size', T(int_or_none)), })): - # bad `fmt` values were `None` and removed - if 'url' not in fmt: - continue - fmt_url = fmt['url'] # known to be valid URL - ext = determine_ext(fmt_url) - if ext == 'm3u8': - fmts.extend(self._extract_m3u8_formats(fmt_url, video_id, 'mp4', fatal=False)) - else: - fmt['ext'] = ext - fmts.append(fmt) - - # sort, raise if no formats - self._sort_formats(fmts) - - info_dict['formats'] = fmts - ... -``` -The extractor raises an exception rather than random crashes if the JSON structure changes so that no formats are found. - # EMBEDDING YOUTUBE-DL youtube-dl makes the best effort to be a good command-line program, and thus should be callable from any programming language. If you encounter any problems parsing its output, feel free to [create a report](https://github.com/ytdl-org/youtube-dl/issues/new). @@ -1497,11 +1367,7 @@ with youtube_dl.YoutubeDL(ydl_opts) as ydl: # BUGS -Bugs and suggestions should be reported in the issue tracker: <https://github.com/ytdl-org/youtube-dl/issues> (<https://yt-dl.org/bug> is an alias for this). Unless you were prompted to or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the IRC channel [#youtube-dl](irc://chat.freenode.net/#youtube-dl) on freenode ([webchat](https://webchat.freenode.net/?randomnick=1&channels=youtube-dl)). - -## Opening a bug report or suggestion - -Be sure to follow instructions provided **below** and **in the issue tracker**. Complete the appropriate issue template fully. Consider whether your problem is covered by an existing issue: if so, follow the discussion there. Avoid commenting on existing duplicate issues as such comments do not add to the discussion of the issue and are liable to be treated as spam. +Bugs and suggestions should be reported at: <https://github.com/ytdl-org/youtube-dl/issues>. Unless you were prompted to or there is another pertinent reason (e.g. GitHub fails to accept the bug report), please do not send bug reports via personal email. For discussions, join us in the IRC channel [#youtube-dl](irc://chat.freenode.net/#youtube-dl) on freenode ([webchat](https://webchat.freenode.net/?randomnick=1&channels=youtube-dl)). **Please include the full output of youtube-dl when run with `-v`**, i.e. **add** `-v` flag to **your command line**, copy the **whole** output and post it in the issue body wrapped in \`\`\` for better formatting. It should look similar to this: ``` @@ -1521,17 +1387,17 @@ $ youtube-dl -v <your command line> The output (including the first lines) contains important debugging information. Issues without the full output are often not reproducible and therefore do not get solved in short order, if ever. -Finally please review your issue to avoid various common mistakes (you can and should use this as a checklist) listed below. +Please re-read your issue once again to avoid a couple of common mistakes (you can and should use this as a checklist): ### Is the description of the issue itself sufficient? -We often get issue reports that are hard to understand. To avoid subsequent clarifications, and to assist participants who are not native English speakers, please elaborate on what feature you are requesting, or what bug you want to be fixed. +We often get issue reports that we cannot really decipher. While in most cases we eventually get the required information after asking back multiple times, this poses an unnecessary drain on our resources. Many contributors, including myself, are also not native speakers, so we may misread some parts. -Make sure that it's obvious +So please elaborate on what feature you are requesting, or what bug you want to be fixed. Make sure that it's obvious - What the problem is - How it could be fixed -- How your proposed solution would look +- How your proposed solution would look like If your report is shorter than two lines, it is almost certainly missing some of these, which makes it hard for us to respond to it. We're often too polite to close the issue outright, but the missing info makes misinterpretation likely. As a committer myself, I often get frustrated by these issues, since the only possible way for me to move forward on them is to ask for clarification over and over. @@ -1541,14 +1407,14 @@ If your server has multiple IPs or you suspect censorship, adding `--call-home` **Site support requests must contain an example URL**. An example URL is a URL you might want to download, like `https://www.youtube.com/watch?v=BaW_jenozKc`. There should be an obvious video present. Except under very special circumstances, the main page of a video service (e.g. `https://www.youtube.com/`) is *not* an example URL. -### Is the issue already documented? - -Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or browse the [GitHub Issues](https://github.com/ytdl-org/youtube-dl/search?type=Issues) of this repository. Initially, at least, use the search term `-label:duplicate` to focus on active issues. If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity. - ### Are you using the latest version? Before reporting any issue, type `youtube-dl -U`. This should report that you're up-to-date. About 20% of the reports we receive are already fixed, but people are using outdated versions. This goes for feature requests as well. +### Is the issue already documented? + +Make sure that someone has not already opened the issue you're trying to open. Search at the top of the window or browse the [GitHub Issues](https://github.com/ytdl-org/youtube-dl/search?type=Issues) of this repository. If there is an issue, feel free to write something along the lines of "This affects me as well, with version 2015.01.01. Here is some more information on the issue: ...". While some issues may be old, a new post into them often spurs rapid activity. + ### Why are existing options not enough? Before requesting a new feature, please have a quick peek at [the list of supported options](https://github.com/ytdl-org/youtube-dl/blob/master/README.md#options). Many feature requests are for features that actually exist already! Please, absolutely do show off your work in the issue report and detail how the existing similar options do *not* solve your problem. diff --git a/devscripts/__init__.py b/devscripts/__init__.py deleted file mode 100644 index 750dbdca7..000000000 --- a/devscripts/__init__.py +++ /dev/null @@ -1 +0,0 @@ -# Empty file needed to make devscripts.utils properly importable from outside diff --git a/devscripts/bash-completion.py b/devscripts/bash-completion.py index 7db396a77..3d1391334 100755 --- a/devscripts/bash-completion.py +++ b/devscripts/bash-completion.py @@ -5,12 +5,8 @@ import os from os.path import dirname as dirn import sys -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) - +sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) import youtube_dl -from youtube_dl.compat import compat_open as open - -from utils import read_file BASH_COMPLETION_FILE = "youtube-dl.bash-completion" BASH_COMPLETION_TEMPLATE = "devscripts/bash-completion.in" @@ -22,8 +18,9 @@ def build_completion(opt_parser): for option in group.option_list: # for every long flag opts_flag.append(option.get_opt_string()) - template = read_file(BASH_COMPLETION_TEMPLATE) - with open(BASH_COMPLETION_FILE, "w", encoding='utf-8') as f: + with open(BASH_COMPLETION_TEMPLATE) as f: + template = f.read() + with open(BASH_COMPLETION_FILE, "w") as f: # just using the special char filled_template = template.replace("{{flags}}", " ".join(opts_flag)) f.write(filled_template) diff --git a/devscripts/cli_to_api.py b/devscripts/cli_to_api.py deleted file mode 100755 index 9fb1d2ba8..000000000 --- a/devscripts/cli_to_api.py +++ /dev/null @@ -1,83 +0,0 @@ -#!/usr/bin/env python -# coding: utf-8 - -from __future__ import unicode_literals - -""" -This script displays the API parameters corresponding to a yt-dl command line - -Example: -$ ./cli_to_api.py -f best -{u'format': 'best'} -$ -""" - -# Allow direct execution -import os -import sys -sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) - -import youtube_dl -from types import MethodType - - -def cli_to_api(*opts): - YDL = youtube_dl.YoutubeDL - - # to extract the parsed options, break out of YoutubeDL instantiation - - # return options via this Exception - class ParseYTDLResult(Exception): - def __init__(self, result): - super(ParseYTDLResult, self).__init__('result') - self.opts = result - - # replacement constructor that raises ParseYTDLResult - def ytdl_init(ydl, ydl_opts): - super(YDL, ydl).__init__(ydl_opts) - raise ParseYTDLResult(ydl_opts) - - # patch in the constructor - YDL.__init__ = MethodType(ytdl_init, YDL) - - # core parser - def parsed_options(argv): - try: - youtube_dl._real_main(list(argv)) - except ParseYTDLResult as result: - return result.opts - - # from https://github.com/yt-dlp/yt-dlp/issues/5859#issuecomment-1363938900 - default = parsed_options([]) - - def neq_opt(a, b): - if a == b: - return False - if a is None and repr(type(object)).endswith(".utils.DateRange'>"): - return '0001-01-01 - 9999-12-31' != '{0}'.format(b) - return a != b - - diff = dict((k, v) for k, v in parsed_options(opts).items() if neq_opt(default[k], v)) - if 'postprocessors' in diff: - diff['postprocessors'] = [pp for pp in diff['postprocessors'] if pp not in default['postprocessors']] - return diff - - -def main(): - from pprint import PrettyPrinter - - pprint = PrettyPrinter() - super_format = pprint.format - - def format(object, context, maxlevels, level): - if repr(type(object)).endswith(".utils.DateRange'>"): - return '{0}: {1}>'.format(repr(object)[:-2], object), True, False - return super_format(object, context, maxlevels, level) - - pprint.format = format - - pprint.pprint(cli_to_api(*sys.argv)) - - -if __name__ == '__main__': - main() diff --git a/devscripts/create-github-release.py b/devscripts/create-github-release.py index 320bcfc27..2ddfa1096 100644 --- a/devscripts/create-github-release.py +++ b/devscripts/create-github-release.py @@ -1,6 +1,7 @@ #!/usr/bin/env python from __future__ import unicode_literals +import io import json import mimetypes import netrc @@ -9,9 +10,7 @@ import os import re import sys -dirn = os.path.dirname - -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) from youtube_dl.compat import ( compat_basestring, @@ -23,7 +22,6 @@ from youtube_dl.utils import ( make_HTTPS_handler, sanitized_Request, ) -from utils import read_file class GitHubReleaser(object): @@ -91,7 +89,8 @@ def main(): changelog_file, version, build_path = args - changelog = read_file(changelog_file) + with io.open(changelog_file, encoding='utf-8') as inf: + changelog = inf.read() mobj = re.search(r'(?s)version %s\n{2}(.+?)\n{3}' % version, changelog) body = mobj.group(1) if mobj else '' diff --git a/devscripts/fish-completion.py b/devscripts/fish-completion.py index ef8a39e0b..51d19dd33 100755 --- a/devscripts/fish-completion.py +++ b/devscripts/fish-completion.py @@ -6,13 +6,10 @@ import os from os.path import dirname as dirn import sys -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) - +sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) import youtube_dl from youtube_dl.utils import shell_quote -from utils import read_file, write_file - FISH_COMPLETION_FILE = 'youtube-dl.fish' FISH_COMPLETION_TEMPLATE = 'devscripts/fish-completion.in' @@ -41,9 +38,11 @@ def build_completion(opt_parser): complete_cmd.extend(EXTRA_ARGS.get(long_option, [])) commands.append(shell_quote(complete_cmd)) - template = read_file(FISH_COMPLETION_TEMPLATE) + with open(FISH_COMPLETION_TEMPLATE) as f: + template = f.read() filled_template = template.replace('{{commands}}', '\n'.join(commands)) - write_file(FISH_COMPLETION_FILE, filled_template) + with open(FISH_COMPLETION_FILE, 'w') as f: + f.write(filled_template) parser = youtube_dl.parseOpts()[0] diff --git a/devscripts/gh-pages/add-version.py b/devscripts/gh-pages/add-version.py index b84908f85..867ea0048 100755 --- a/devscripts/gh-pages/add-version.py +++ b/devscripts/gh-pages/add-version.py @@ -6,21 +6,16 @@ import sys import hashlib import os.path -dirn = os.path.dirname - -sys.path.insert(0, dirn(dirn(dirn(os.path.abspath(__file__))))) - -from devscripts.utils import read_file, write_file -from youtube_dl.compat import compat_open as open if len(sys.argv) <= 1: print('Specify the version number as parameter') sys.exit() version = sys.argv[1] -write_file('update/LATEST_VERSION', version) +with open('update/LATEST_VERSION', 'w') as f: + f.write(version) -versions_info = json.loads(read_file('update/versions.json')) +versions_info = json.load(open('update/versions.json')) if 'signature' in versions_info: del versions_info['signature'] @@ -44,5 +39,5 @@ for key, filename in filenames.items(): versions_info['versions'][version] = new_version versions_info['latest'] = version -with open('update/versions.json', 'w', encoding='utf-8') as jsonf: - json.dumps(versions_info, jsonf, indent=4, sort_keys=True) +with open('update/versions.json', 'w') as jsonf: + json.dump(versions_info, jsonf, indent=4, sort_keys=True) diff --git a/devscripts/gh-pages/generate-download.py b/devscripts/gh-pages/generate-download.py index 3e38e9299..a873d32ee 100755 --- a/devscripts/gh-pages/generate-download.py +++ b/devscripts/gh-pages/generate-download.py @@ -2,21 +2,14 @@ from __future__ import unicode_literals import json -import os.path -import sys -dirn = os.path.dirname - -sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) - -from utils import read_file, write_file - -versions_info = json.loads(read_file('update/versions.json')) +versions_info = json.load(open('update/versions.json')) version = versions_info['latest'] version_dict = versions_info['versions'][version] # Read template page -template = read_file('download.html.in') +with open('download.html.in', 'r', encoding='utf-8') as tmplf: + template = tmplf.read() template = template.replace('@PROGRAM_VERSION@', version) template = template.replace('@PROGRAM_URL@', version_dict['bin'][0]) @@ -25,5 +18,5 @@ template = template.replace('@EXE_URL@', version_dict['exe'][0]) template = template.replace('@EXE_SHA256SUM@', version_dict['exe'][1]) template = template.replace('@TAR_URL@', version_dict['tar'][0]) template = template.replace('@TAR_SHA256SUM@', version_dict['tar'][1]) - -write_file('download.html', template) +with open('download.html', 'w', encoding='utf-8') as dlf: + dlf.write(template) diff --git a/devscripts/gh-pages/update-copyright.py b/devscripts/gh-pages/update-copyright.py index 444595c48..61487f925 100755 --- a/devscripts/gh-pages/update-copyright.py +++ b/devscripts/gh-pages/update-copyright.py @@ -5,22 +5,17 @@ from __future__ import with_statement, unicode_literals import datetime import glob +import io # For Python 2 compatibility import os import re -import sys -dirn = os.path.dirname - -sys.path.insert(0, dirn(dirn(dirn(os.path.abspath(__file__))))) - -from devscripts.utils import read_file, write_file -from youtube_dl import compat_str - -year = compat_str(datetime.datetime.now().year) +year = str(datetime.datetime.now().year) for fn in glob.glob('*.html*'): - content = read_file(fn) + with io.open(fn, encoding='utf-8') as f: + content = f.read() newc = re.sub(r'(?P<copyright>Copyright © 2011-)(?P<year>[0-9]{4})', 'Copyright © 2011-' + year, content) if content != newc: tmpFn = fn + '.part' - write_file(tmpFn, newc) + with io.open(tmpFn, 'wt', encoding='utf-8') as outf: + outf.write(newc) os.rename(tmpFn, fn) diff --git a/devscripts/gh-pages/update-feed.py b/devscripts/gh-pages/update-feed.py index 13a367d34..506a62377 100755 --- a/devscripts/gh-pages/update-feed.py +++ b/devscripts/gh-pages/update-feed.py @@ -2,16 +2,10 @@ from __future__ import unicode_literals import datetime +import io import json -import os.path import textwrap -import sys -dirn = os.path.dirname - -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) - -from utils import write_file atom_template = textwrap.dedent("""\ <?xml version="1.0" encoding="utf-8"?> @@ -78,4 +72,5 @@ for v in versions: entries_str = textwrap.indent(''.join(entries), '\t') atom_template = atom_template.replace('@ENTRIES@', entries_str) -write_file('update/releases.atom', atom_template) +with io.open('update/releases.atom', 'w', encoding='utf-8') as atom_file: + atom_file.write(atom_template) diff --git a/devscripts/gh-pages/update-sites.py b/devscripts/gh-pages/update-sites.py index 06a8a474c..531c93c70 100755 --- a/devscripts/gh-pages/update-sites.py +++ b/devscripts/gh-pages/update-sites.py @@ -5,17 +5,15 @@ import sys import os import textwrap -dirn = os.path.dirname - # We must be able to import youtube_dl -sys.path.insert(0, dirn(dirn(dirn(os.path.abspath(__file__))))) +sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))) import youtube_dl -from devscripts.utils import read_file, write_file def main(): - template = read_file('supportedsites.html.in') + with open('supportedsites.html.in', 'r', encoding='utf-8') as tmplf: + template = tmplf.read() ie_htmls = [] for ie in youtube_dl.list_extractors(age_limit=None): @@ -31,7 +29,8 @@ def main(): template = template.replace('@SITES@', textwrap.indent('\n'.join(ie_htmls), '\t')) - write_file('supportedsites.html', template) + with open('supportedsites.html', 'w', encoding='utf-8') as sitesf: + sitesf.write(template) if __name__ == '__main__': diff --git a/devscripts/install_jython.sh b/devscripts/install_jython.sh new file mode 100755 index 000000000..bafca4da4 --- /dev/null +++ b/devscripts/install_jython.sh @@ -0,0 +1,5 @@ +#!/bin/bash + +wget http://central.maven.org/maven2/org/python/jython-installer/2.7.1/jython-installer-2.7.1.jar +java -jar jython-installer-2.7.1.jar -s -d "$HOME/jython" +$HOME/jython/bin/jython -m pip install nose diff --git a/devscripts/make_contributing.py b/devscripts/make_contributing.py index 5a9eb194f..226d1a5d6 100755 --- a/devscripts/make_contributing.py +++ b/devscripts/make_contributing.py @@ -1,11 +1,10 @@ #!/usr/bin/env python from __future__ import unicode_literals +import io import optparse import re -from utils import read_file, write_file - def main(): parser = optparse.OptionParser(usage='%prog INFILE OUTFILE') @@ -15,7 +14,8 @@ def main(): infile, outfile = args - readme = read_file(infile) + with io.open(infile, encoding='utf-8') as inf: + readme = inf.read() bug_text = re.search( r'(?s)#\s*BUGS\s*[^\n]*\s*(.*?)#\s*COPYRIGHT', readme).group(1) @@ -25,7 +25,8 @@ def main(): out = bug_text + dev_text - write_file(outfile, out) + with io.open(outfile, 'w', encoding='utf-8') as outf: + outf.write(out) if __name__ == '__main__': diff --git a/devscripts/make_issue_template.py b/devscripts/make_issue_template.py index 65fa8169f..b7ad23d83 100644 --- a/devscripts/make_issue_template.py +++ b/devscripts/make_issue_template.py @@ -1,11 +1,8 @@ #!/usr/bin/env python from __future__ import unicode_literals +import io import optparse -import os.path -import sys - -from utils import read_file, read_version, write_file def main(): @@ -16,11 +13,17 @@ def main(): infile, outfile = args - issue_template_tmpl = read_file(infile) + with io.open(infile, encoding='utf-8') as inf: + issue_template_tmpl = inf.read() - out = issue_template_tmpl % {'version': read_version()} + # Get the version from youtube_dl/version.py without importing the package + exec(compile(open('youtube_dl/version.py').read(), + 'youtube_dl/version.py', 'exec')) - write_file(outfile, out) + out = issue_template_tmpl % {'version': locals()['__version__']} + + with io.open(outfile, 'w', encoding='utf-8') as outf: + outf.write(out) if __name__ == '__main__': main() diff --git a/devscripts/make_lazy_extractors.py b/devscripts/make_lazy_extractors.py index 5b8b123a4..0a1762dbc 100644 --- a/devscripts/make_lazy_extractors.py +++ b/devscripts/make_lazy_extractors.py @@ -1,49 +1,28 @@ from __future__ import unicode_literals, print_function from inspect import getsource +import io import os from os.path import dirname as dirn -import re import sys print('WARNING: Lazy loading extractors is an experimental feature that may not always work', file=sys.stderr) -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) +sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) lazy_extractors_filename = sys.argv[1] if os.path.exists(lazy_extractors_filename): os.remove(lazy_extractors_filename) -# Py2: may be confused by leftover lazy_extractors.pyc -if sys.version_info[0] < 3: - for c in ('c', 'o'): - try: - os.remove(lazy_extractors_filename + 'c') - except OSError: - pass - -from devscripts.utils import read_file, write_file -from youtube_dl.compat import compat_register_utf8 - -compat_register_utf8() from youtube_dl.extractor import _ALL_CLASSES from youtube_dl.extractor.common import InfoExtractor, SearchInfoExtractor -module_template = read_file('devscripts/lazy_load_template.py') - - -def get_source(m): - return re.sub(r'(?m)^\s*#.*\n', '', getsource(m)) - +with open('devscripts/lazy_load_template.py', 'rt') as f: + module_template = f.read() module_contents = [ - module_template, - get_source(InfoExtractor.suitable), - get_source(InfoExtractor._match_valid_url) + '\n', - 'class LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n', - # needed for suitable() methods of Youtube extractor (see #28780) - 'from youtube_dl.utils import parse_qs, variadic\n', -] + module_template + '\n' + getsource(InfoExtractor.suitable) + '\n', + 'class LazyLoadSearchExtractor(LazyLoadExtractor):\n pass\n'] ie_template = ''' class {name}({bases}): @@ -75,14 +54,14 @@ def build_lazy_ie(ie, name): valid_url=valid_url, module=ie.__module__) if ie.suitable.__func__ is not InfoExtractor.suitable.__func__: - s += '\n' + get_source(ie.suitable) + s += '\n' + getsource(ie.suitable) if hasattr(ie, '_make_valid_url'): # search extractors s += make_valid_template.format(valid_url=ie._make_valid_url()) return s -# find the correct sorting and add the required base classes so that subclasses +# find the correct sorting and add the required base classes so that sublcasses # can be correctly created classes = _ALL_CLASSES[:-1] ordered_cls = [] @@ -115,17 +94,7 @@ for ie in ordered_cls: module_contents.append( '_ALL_CLASSES = [{0}]'.format(', '.join(names))) -module_src = '\n'.join(module_contents) +module_src = '\n'.join(module_contents) + '\n' -write_file(lazy_extractors_filename, module_src + '\n') - -# work around JVM byte code module limit in Jython -if sys.platform.startswith('java') and sys.version_info[:2] == (2, 7): - import subprocess - from youtube_dl.compat import compat_subprocess_get_DEVNULL - # if Python 2.7 is available, use it to compile the module for Jython - try: - # if Python 2.7 is available, use it to compile the module for Jython - subprocess.check_call(['python2.7', '-m', 'py_compile', lazy_extractors_filename], stdout=compat_subprocess_get_DEVNULL()) - except Exception: - pass +with io.open(lazy_extractors_filename, 'wt', encoding='utf-8') as f: + f.write(module_src) diff --git a/devscripts/make_readme.py b/devscripts/make_readme.py index 7a5b04dcc..8fbce0796 100755 --- a/devscripts/make_readme.py +++ b/devscripts/make_readme.py @@ -1,14 +1,8 @@ from __future__ import unicode_literals -import os.path -import re +import io import sys -dirn = os.path.dirname - -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) - -from utils import read_file -from youtube_dl.compat import compat_open as open +import re README_FILE = 'README.md' helptext = sys.stdin.read() @@ -16,7 +10,8 @@ helptext = sys.stdin.read() if isinstance(helptext, bytes): helptext = helptext.decode('utf-8') -oldreadme = read_file(README_FILE) +with io.open(README_FILE, encoding='utf-8') as f: + oldreadme = f.read() header = oldreadme[:oldreadme.index('# OPTIONS')] footer = oldreadme[oldreadme.index('# CONFIGURATION'):] @@ -25,7 +20,7 @@ options = helptext[helptext.index(' General Options:') + 19:] options = re.sub(r'(?m)^ (\w.+)$', r'## \1', options) options = '# OPTIONS\n' + options + '\n' -with open(README_FILE, 'w', encoding='utf-8') as f: +with io.open(README_FILE, 'w', encoding='utf-8') as f: f.write(header) f.write(options) f.write(footer) diff --git a/devscripts/make_supportedsites.py b/devscripts/make_supportedsites.py index c424d18d7..764795bc5 100644 --- a/devscripts/make_supportedsites.py +++ b/devscripts/make_supportedsites.py @@ -1,19 +1,17 @@ #!/usr/bin/env python from __future__ import unicode_literals +import io import optparse -import os.path +import os import sys + # Import youtube_dl -dirn = os.path.dirname - -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) - +ROOT_DIR = os.path.join(os.path.dirname(__file__), '..') +sys.path.insert(0, ROOT_DIR) import youtube_dl -from utils import write_file - def main(): parser = optparse.OptionParser(usage='%prog OUTFILE.md') @@ -40,7 +38,8 @@ def main(): ' - ' + md + '\n' for md in gen_ies_md(ies)) - write_file(outfile, out) + with io.open(outfile, 'w', encoding='utf-8') as outf: + outf.write(out) if __name__ == '__main__': diff --git a/devscripts/prepare_manpage.py b/devscripts/prepare_manpage.py index 0090ada3e..76bf873e1 100644 --- a/devscripts/prepare_manpage.py +++ b/devscripts/prepare_manpage.py @@ -1,13 +1,13 @@ from __future__ import unicode_literals +import io import optparse import os.path import re -from utils import read_file, write_file - ROOT_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) README_FILE = os.path.join(ROOT_DIR, 'README.md') + PREFIX = r'''%YOUTUBE-DL(1) # NAME @@ -29,7 +29,8 @@ def main(): outfile, = args - readme = read_file(README_FILE) + with io.open(README_FILE, encoding='utf-8') as f: + readme = f.read() readme = re.sub(r'(?s)^.*?(?=# DESCRIPTION)', '', readme) readme = re.sub(r'\s+youtube-dl \[OPTIONS\] URL \[URL\.\.\.\]', '', readme) @@ -37,7 +38,8 @@ def main(): readme = filter_options(readme) - write_file(outfile, readme) + with io.open(outfile, 'w', encoding='utf-8') as outf: + outf.write(readme) def filter_options(readme): diff --git a/devscripts/run_tests.bat b/devscripts/run_tests.bat deleted file mode 100644 index 79359b5a7..000000000 --- a/devscripts/run_tests.bat +++ /dev/null @@ -1,17 +0,0 @@ -@echo off - -rem Keep this list in sync with the `offlinetest` target in Makefile -set DOWNLOAD_TESTS="age_restriction^|download^|iqiyi_sdk_interpreter^|socks^|subtitles^|write_annotations^|youtube_lists^|youtube_signature" - -if "%YTDL_TEST_SET%" == "core" ( - set test_set="-I test_("%DOWNLOAD_TESTS%")\.py" - set multiprocess_args="" -) else if "%YTDL_TEST_SET%" == "download" ( - set test_set="-I test_(?!"%DOWNLOAD_TESTS%").+\.py" - set multiprocess_args="--processes=4 --process-timeout=540" -) else ( - echo YTDL_TEST_SET is not set or invalid - exit /b 1 -) - -nosetests test --verbose %test_set:"=% %multiprocess_args:"=% diff --git a/devscripts/utils.py b/devscripts/utils.py deleted file mode 100644 index 2d072d2e0..000000000 --- a/devscripts/utils.py +++ /dev/null @@ -1,62 +0,0 @@ -# coding: utf-8 -from __future__ import unicode_literals - -import argparse -import functools -import os.path -import subprocess -import sys - -dirn = os.path.dirname - -sys.path.insert(0, dirn(dirn(os.path.abspath(__file__)))) - -from youtube_dl.compat import ( - compat_kwargs, - compat_open as open, -) - - -def read_file(fname): - with open(fname, encoding='utf-8') as f: - return f.read() - - -def write_file(fname, content, mode='w'): - with open(fname, mode, encoding='utf-8') as f: - return f.write(content) - - -def read_version(fname='youtube_dl/version.py'): - """Get the version without importing the package""" - exec(compile(read_file(fname), fname, 'exec')) - return locals()['__version__'] - - -def get_filename_args(has_infile=False, default_outfile=None): - parser = argparse.ArgumentParser() - if has_infile: - parser.add_argument('infile', help='Input file') - kwargs = {'nargs': '?', 'default': default_outfile} if default_outfile else {} - kwargs['help'] = 'Output file' - parser.add_argument('outfile', **compat_kwargs(kwargs)) - - opts = parser.parse_args() - if has_infile: - return opts.infile, opts.outfile - return opts.outfile - - -def compose_functions(*functions): - return lambda x: functools.reduce(lambda y, f: f(y), functions, x) - - -def run_process(*args, **kwargs): - kwargs.setdefault('text', True) - kwargs.setdefault('check', True) - kwargs.setdefault('capture_output', True) - if kwargs['text']: - kwargs.setdefault('encoding', 'utf-8') - kwargs.setdefault('errors', 'replace') - kwargs = compat_kwargs(kwargs) - return subprocess.run(args, **kwargs) diff --git a/devscripts/zsh-completion.py b/devscripts/zsh-completion.py index ebd552fcb..60aaf76cc 100755 --- a/devscripts/zsh-completion.py +++ b/devscripts/zsh-completion.py @@ -7,8 +7,6 @@ import sys sys.path.insert(0, dirn(dirn((os.path.abspath(__file__))))) import youtube_dl -from utils import read_file, write_file - ZSH_COMPLETION_FILE = "youtube-dl.zsh" ZSH_COMPLETION_TEMPLATE = "devscripts/zsh-completion.in" @@ -36,13 +34,15 @@ def build_completion(opt_parser): flags = [opt.get_opt_string() for opt in opts] - template = read_file(ZSH_COMPLETION_TEMPLATE) + with open(ZSH_COMPLETION_TEMPLATE) as f: + template = f.read() template = template.replace("{{fileopts}}", "|".join(fileopts)) template = template.replace("{{diropts}}", "|".join(diropts)) template = template.replace("{{flags}}", " ".join(flags)) - write_file(ZSH_COMPLETION_FILE, template) + with open(ZSH_COMPLETION_FILE, "w") as f: + f.write(template) parser = youtube_dl.parseOpts()[0] diff --git a/docs/supportedsites.md b/docs/supportedsites.md index ae2a6b8b0..367545a96 100644 --- a/docs/supportedsites.md +++ b/docs/supportedsites.md @@ -1,9 +1,9 @@ # Supported sites - **1tv**: Первый канал + - **1up.com** - **20min** - **220.ro** - **23video** - - **247sports** - **24video** - **3qsdn**: 3Q SDN - **3sat** @@ -35,39 +35,32 @@ - **adobetv:video** - **AdultSwim** - **aenetworks**: A+E Networks: A&E, Lifetime, History.com, FYI Network and History Vault - - **aenetworks:collection** - - **aenetworks:show** - **afreecatv**: afreecatv.com - **AirMozilla** - **AliExpressLive** - **AlJazeera** - **Allocine** - **AlphaPorno** - - **Amara** - **AMCNetworks** - **AmericasTestKitchen** - - **AmericasTestKitchenSeason** - **anderetijden**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl - **AnimeOnDemand** - **Anvato** - - **aol.com**: Yahoo screen and movies + - **aol.com** - **APA** - **Aparat** - **AppleConnect** - **AppleDaily**: 臺灣蘋果日報 - - **ApplePodcasts** - **appletrailers** - **appletrailers:section** - **archive.org**: archive.org videos - - **ArcPublishing** - **ARD** - **ARD:mediathek** - **ARDBetaMediathek** - **Arkena** - - **arte.sky.it** - - **ArteTV** - - **ArteTVEmbed** - - **ArteTVPlaylist** + - **arte.tv:+7** + - **arte.tv:embed** + - **arte.tv:playlist** - **AsianCrush** - **AsianCrushPlaylist** - **AtresPlayer** @@ -83,7 +76,6 @@ - **awaan:video** - **AZMedien**: AZ Medien videos - **BaiduVideo**: 百度视频 - - **bandaichannel** - **Bandcamp** - **Bandcamp:album** - **Bandcamp:weekly** @@ -91,8 +83,7 @@ - **bbc**: BBC - **bbc.co.uk**: BBC iPlayer - **bbc.co.uk:article**: BBC articles - - **bbc.co.uk:iplayer:episodes** - - **bbc.co.uk:iplayer:group** + - **bbc.co.uk:iplayer:playlist** - **bbc.co.uk:playlist** - **BBVTV** - **Beatport** @@ -102,10 +93,6 @@ - **BellMedia** - **Bet** - **bfi:player** - - **bfmtv** - - **bfmtv:article** - - **bfmtv:live** - - **BibelTV** - **Bigflix** - **Bild**: Bild.de - **BiliBili** @@ -113,17 +100,15 @@ - **BilibiliAudioAlbum** - **BiliBiliPlayer** - **BioBioChileTV** - - **Biography** - **BIQLE** - **BitChute** - **BitChuteChannel** - **BleacherReport** - **BleacherReportCMS** + - **blinkx** - **Bloomberg** - **BokeCC** - - **BongaCams** - **BostonGlobe** - - **Box** - **Bpb**: Bundeszentrale für politische Bildung - **BR**: Bayerischer Rundfunk - **BravoTV** @@ -156,12 +141,10 @@ - **CBS** - **CBSInteractive** - **CBSLocal** - - **CBSLocalArticle** - **cbsnews**: CBS News - **cbsnews:embed** - **cbsnews:livevideo**: CBS News Live Videos - - **cbssports** - - **cbssports:embed** + - **CBSSports** - **CCMA** - **CCTV**: 央视网 - **CDA** @@ -173,7 +156,6 @@ - **Chilloutzone** - **chirbit** - **chirbit:profile** - - **cielotv.it** - **Cinchcast** - **Cinemax** - **CiscoLiveSearch** @@ -195,6 +177,8 @@ - **CNNArticle** - **CNNBlogs** - **ComedyCentral** + - **ComedyCentralFullEpisodes** + - **ComedyCentralShortname** - **ComedyCentralTV** - **CondeNast**: Condé Nast media group: Allure, Architectural Digest, Ars Technica, Bon Appétit, Brides, Condé Nast, Condé Nast Traveler, Details, Epicurious, GQ, Glamour, Golf Digest, SELF, Teen Vogue, The New Yorker, Vanity Fair, Vogue, W Magazine, WIRED - **CONtv** @@ -205,9 +189,9 @@ - **CrooksAndLiars** - **crunchyroll** - **crunchyroll:playlist** + - **CSNNE** - **CSpan**: C-SPAN - **CtsNews**: 華視新聞 - - **CTV** - **CTVNews** - **cu.ntv.co.jp**: Nippon Television Network - **Culturebox** @@ -215,7 +199,6 @@ - **curiositystream** - **curiositystream:collection** - **CWTV** - - **DagelijkseKost**: dagelijksekost.een.be - **DailyMail** - **dailymotion** - **dailymotion:playlist** @@ -237,7 +220,6 @@ - **DiscoveryGo** - **DiscoveryGoPlaylist** - **DiscoveryNetworksDe** - - **DiscoveryPlus** - **DiscoveryVR** - **Disney** - **dlive:stream** @@ -280,6 +262,7 @@ - **ESPNArticle** - **EsriVideo** - **Europa** + - **EveryonesMixtape** - **EWETV** - **ExpoTV** - **Expressen** @@ -321,11 +304,11 @@ - **FrontendMasters** - **FrontendMastersCourse** - **FrontendMastersLesson** - - **FujiTVFODPlus7** - **Funimation** - **Funk** - **Fusion** - **Fux** + - **FXNetworks** - **Gaia** - **GameInformer** - **GameSpot** @@ -333,7 +316,6 @@ - **Gaskrank** - **Gazeta** - **GDCVault** - - **GediDigital** - **generic**: Generic downloader that works on some sites - **Gfycat** - **GiantBomb** @@ -345,8 +327,6 @@ - **Go** - **GodTube** - **Golem** - - **google:podcasts** - - **google:podcasts:feed** - **GoogleDrive** - **Goshgay** - **GPUTechConf** @@ -359,10 +339,8 @@ - **HentaiStigma** - **hetklokhuis** - **hgtv.com:show** - - **HGTVDe** - **HiDive** - **HistoricFilms** - - **history:player** - **history:topic**: History.com Topic - **hitbox** - **hitbox:live** @@ -382,10 +360,6 @@ - **HungamaSong** - **Hypem** - **ign.com** - - **IGNArticle** - - **IGNVideo** - - **IHeartRadio** - - **iheartradio:podcast** - **imdb**: Internet Movie Database trailers - **imdb:list**: Internet Movie Database lists - **Imgur** @@ -419,14 +393,14 @@ - **JWPlatform** - **Kakao** - **Kaltura** + - **KanalPlay**: Kanal 5/9/11 Play - **Kankan** - **Karaoketv** - **KarriereVideos** - **Katsomo** - **KeezMovies** - **Ketnet** - - **khanacademy** - - **khanacademy:unit** + - **KhanAcademy** - **KickStarter** - **KinjaEmbed** - **KinoPoisk** @@ -443,8 +417,6 @@ - **la7.it** - **laola1tv** - **laola1tv:embed** - - **lbry** - - **lbry:channel** - **LCI** - **Lcp** - **LcpPlay** @@ -464,14 +436,14 @@ - **limelight** - **limelight:channel** - **limelight:channel_list** - - **LineLive** - - **LineLiveChannel** - **LineTV** - **linkedin:learning** - **linkedin:learning:course** - **LinuxAcademy** - **LiTV** - **LiveJournal** + - **LiveLeak** + - **LiveLeakEmbed** - **livestream** - **livestream:original** - **LnkGo** @@ -489,13 +461,11 @@ - **mangomolo:live** - **mangomolo:video** - **ManyVids** - - **MaoriTV** - **Markiza** - **MarkizaPage** - **massengeschmack.tv** - **MatchTV** - **MDR**: MDR.DE and KiKA - - **MedalTV** - **media.ccc.de** - **media.ccc.de:lists** - **Medialaan** @@ -510,13 +480,9 @@ - **META** - **metacafe** - **Metacritic** - - **mewatch** - **Mgoon** - **MGTV**: 芒果TV - **MiaoPai** - - **minds** - - **minds:channel** - - **minds:group** - **MinistryGrid** - **Minoto** - **miomio.tv** @@ -524,8 +490,9 @@ - **mixcloud** - **mixcloud:playlist** - **mixcloud:user** + - **Mixer:live** + - **Mixer:vod** - **MLB** - - **MLBVideo** - **Mnet** - **MNetTV** - **MoeVideo**: LetitBit video services: moevideo.net, playreplay.net and videochart.net @@ -547,7 +514,6 @@ - **mtv:video** - **mtvjapan** - **mtvservices:embedded** - - **MTVUutisetArticle** - **MuenchenTV**: münchen.tv - **mva**: Microsoft Virtual Academy videos - **mva:course**: Microsoft Virtual Academy courses @@ -566,11 +532,6 @@ - **NationalGeographicTV** - **Naver** - **NBA** - - **nba:watch** - - **nba:watch:collection** - - **NBAChannel** - - **NBAEmbed** - - **NBAWatchEmbed** - **NBC** - **NBCNews** - **nbcolympics** @@ -600,10 +561,8 @@ - **NextTV**: 壹電視 - **Nexx** - **NexxEmbed** - - **nfl.com** (Currently broken) - - **nfl.com:article** (Currently broken) + - **nfl.com** - **NhkVod** - - **NhkVodProgram** - **nhl.com** - **nick.com** - **nick.de** @@ -617,6 +576,7 @@ - **njoy:embed** - **NJPWWorld**: 新日本プロレスワールド - **NobelPrize** + - **Noco** - **NonkTube** - **Noovo** - **Normalboots** @@ -634,7 +594,6 @@ - **Npr** - **NRK** - **NRKPlaylist** - - **NRKRadioPodkast** - **NRKSkole**: NRK Skole - **NRKTV**: NRK TV and NRK Radio - **NRKTVDirekte**: NRK TV Direkte and NRK Radio Direkte @@ -647,7 +606,6 @@ - **Nuvid** - **NYTimes** - **NYTimesArticle** - - **NYTimesCooking** - **NZZ** - **ocw.mit.edu** - **OdaTV** @@ -681,14 +639,12 @@ - **OutsideTV** - **PacktPub** - **PacktPubCourse** - - **PalcoMP3:artist** - - **PalcoMP3:song** - - **PalcoMP3:video** - **pandora.tv**: 판도라TV - **ParamountNetwork** - **parliamentlive.tv**: UK parliament videos - **Patreon** - **pbs**: Public Broadcasting Service (PBS) and member stations: PBS: Public Broadcasting Service, APT - Alabama Public Television (WBIQ), GPB/Georgia Public Broadcasting (WGTV), Mississippi Public Broadcasting (WMPN), Nashville Public Television (WNPT), WFSU-TV (WFSU), WSRE (WSRE), WTCI (WTCI), WPBA/Channel 30 (WPBA), Alaska Public Media (KAKM), Arizona PBS (KAET), KNME-TV/Channel 5 (KNME), Vegas PBS (KLVX), AETN/ARKANSAS ETV NETWORK (KETS), KET (WKLE), WKNO/Channel 10 (WKNO), LPB/LOUISIANA PUBLIC BROADCASTING (WLPB), OETA (KETA), Ozarks Public Television (KOZK), WSIU Public Broadcasting (WSIU), KEET TV (KEET), KIXE/Channel 9 (KIXE), KPBS San Diego (KPBS), KQED (KQED), KVIE Public Television (KVIE), PBS SoCal/KOCE (KOCE), ValleyPBS (KVPT), CONNECTICUT PUBLIC TELEVISION (WEDH), KNPB Channel 5 (KNPB), SOPTV (KSYS), Rocky Mountain PBS (KRMA), KENW-TV3 (KENW), KUED Channel 7 (KUED), Wyoming PBS (KCWC), Colorado Public Television / KBDI 12 (KBDI), KBYU-TV (KBYU), Thirteen/WNET New York (WNET), WGBH/Channel 2 (WGBH), WGBY (WGBY), NJTV Public Media NJ (WNJT), WLIW21 (WLIW), mpt/Maryland Public Television (WMPB), WETA Television and Radio (WETA), WHYY (WHYY), PBS 39 (WLVT), WVPT - Your Source for PBS and More! (WVPT), Howard University Television (WHUT), WEDU PBS (WEDU), WGCU Public Media (WGCU), WPBT2 (WPBT), WUCF TV (WUCF), WUFT/Channel 5 (WUFT), WXEL/Channel 42 (WXEL), WLRN/Channel 17 (WLRN), WUSF Public Broadcasting (WUSF), ETV (WRLK), UNC-TV (WUNC), PBS Hawaii - Oceanic Cable Channel 10 (KHET), Idaho Public Television (KAID), KSPS (KSPS), OPB (KOPB), KWSU/Channel 10 & KTNW/Channel 31 (KWSU), WILL-TV (WILL), Network Knowledge - WSEC/Springfield (WSEC), WTTW11 (WTTW), Iowa Public Television/IPTV (KDIN), Nine Network (KETC), PBS39 Fort Wayne (WFWA), WFYI Indianapolis (WFYI), Milwaukee Public Television (WMVS), WNIN (WNIN), WNIT Public Television (WNIT), WPT (WPNE), WVUT/Channel 22 (WVUT), WEIU/Channel 51 (WEIU), WQPT-TV (WQPT), WYCC PBS Chicago (WYCC), WIPB-TV (WIPB), WTIU (WTIU), CET (WCET), ThinkTVNetwork (WPTD), WBGU-TV (WBGU), WGVU TV (WGVU), NET1 (KUON), Pioneer Public Television (KWCM), SDPB Television (KUSD), TPT (KTCA), KSMQ (KSMQ), KPTS/Channel 8 (KPTS), KTWU/Channel 11 (KTWU), East Tennessee PBS (WSJK), WCTE-TV (WCTE), WLJT, Channel 11 (WLJT), WOSU TV (WOSU), WOUB/WOUC (WOUB), WVPB (WVPB), WKYU-PBS (WKYU), KERA 13 (KERA), MPBN (WCBB), Mountain Lake PBS (WCFE), NHPTV (WENH), Vermont PBS (WETK), witf (WITF), WQED Multimedia (WQED), WMHT Educational Telecommunications (WMHT), Q-TV (WDCQ), WTVS Detroit Public TV (WTVS), CMU Public Television (WCMU), WKAR-TV (WKAR), WNMU-TV Public TV 13 (WNMU), WDSE - WRPT (WDSE), WGTE TV (WGTE), Lakeland Public Television (KAWE), KMOS-TV - Channels 6.1, 6.2 and 6.3 (KMOS), MontanaPBS (KUSM), KRWG/Channel 22 (KRWG), KACV (KACV), KCOS/Channel 13 (KCOS), WCNY/Channel 24 (WCNY), WNED (WNED), WPBS (WPBS), WSKG Public TV (WSKG), WXXI (WXXI), WPSU (WPSU), WVIA Public Media Studios (WVIA), WTVI (WTVI), Western Reserve PBS (WNEO), WVIZ/PBS ideastream (WVIZ), KCTS 9 (KCTS), Basin PBS (KPBT), KUHT / Channel 8 (KUHT), KLRN (KLRN), KLRU (KLRU), WTJX Channel 12 (WTJX), WCVE PBS (WCVE), KBTC Public Television (KBTC) + - **pcmag** - **PearVideo** - **PeerTube** - **People** @@ -702,21 +658,18 @@ - **PicartoVod** - **Piksel** - **Pinkbike** - - **Pinterest** - - **PinterestCollection** - **Pladform** - **Platzi** - **PlatziCourse** - **play.fm** - - **player.sky.it** - **PlayPlusTV** - - **PlayStuff** - **PlaysTV** - **Playtvak**: Playtvak.cz, iDNES.cz and Lidovky.cz - **Playvid** - **Playwire** - **pluralsight** - **pluralsight:course** + - **plus.google**: Google Plus - **podomatic** - **Pokemon** - **PolskieRadio** @@ -746,7 +699,6 @@ - **qqmusic:singer**: QQ音乐 - 歌手 - **qqmusic:toplist**: QQ音乐 - 排行榜 - **QuantumTV** - - **Qub** - **Quickline** - **QuicklineLive** - **R7** @@ -801,7 +753,6 @@ - **RTVNH** - **RTVS** - **RUHD** - - **RumbleEmbed** - **rutube**: Rutube videos - **rutube:channel**: Rutube channels - **rutube:embed**: Rutube embedded videos @@ -816,7 +767,6 @@ - **safari:course**: safaribooksonline.com online courses - **SAKTV** - **SaltTV** - - **SampleFocus** - **Sapo**: SAPO Vídeos - **savefrom.net** - **SBS**: sbs.com.au @@ -839,21 +789,19 @@ - **ShahidShow** - **Shared**: shared.sx - **ShowRoomLive** - - **simplecast** - - **simplecast:episode** - - **simplecast:podcast** - **Sina** - - **sky.it** - - **sky:news** - - **sky:sports** - - **sky:sports:news** - - **skyacademy.it** - **SkylineWebcams** + - **SkyNews** - **skynewsarabia:article** - **skynewsarabia:video** + - **SkySports** - **Slideshare** - **SlidesLive** - **Slutload** + - **smotri**: Smotri.com + - **smotri:broadcast**: Smotri.com broadcasts + - **smotri:community**: Smotri.com community videos + - **smotri:user**: Smotri.com user videos - **Snotr** - **Sohu** - **SonyLIV** @@ -875,16 +823,12 @@ - **SpankBangPlaylist** - **Spankwire** - **Spiegel** + - **Spiegel:Article**: Articles on spiegel.de + - **Spiegeltv** - **sport.francetvinfo.fr** - **Sport5** - **SportBox** - **SportDeutschland** - - **spotify** - - **spotify:show** - - **Spreaker** - - **SpreakerPage** - - **SpreakerShow** - - **SpreakerShowPage** - **SpringboardPlatform** - **Sprout** - **sr:mediathek**: Saarländischer Rundfunk @@ -893,10 +837,6 @@ - **stanfordoc**: Stanford Open ClassRoom - **Steam** - **Stitcher** - - **StitcherShow** - - **StoryFire** - - **StoryFireSeries** - - **StoryFireUser** - **Streamable** - **streamcloud.eu** - **StreamCZ** @@ -917,6 +857,7 @@ - **Tagesschau** - **tagesschau:player** - **Tass** + - **TastyTrade** - **TBS** - **TDSLifeway** - **Teachable** @@ -939,7 +880,6 @@ - **TeleQuebecEmission** - **TeleQuebecLive** - **TeleQuebecSquat** - - **TeleQuebecVideo** - **TeleTask** - **Telewebion** - **TennisTV** @@ -957,7 +897,7 @@ - **ThisAV** - **ThisOldHouse** - **TikTok** - - **TikTokUser** (Currently broken) + - **TikTokUser** - **tinypic**: tinypic.com videos - **TMZ** - **TMZArticle** @@ -965,13 +905,12 @@ - **TNAFlixNetworkEmbed** - **toggle** - **ToonGoggles** + - **Tosh**: Tosh.0 - **tou.tv** - **Toypics**: Toypics video - **ToypicsUser**: Toypics user profile - **TrailerAddict** (Currently broken) - **Trilulilu** - - **Trovo** - - **TrovoVod** - **TruNews** - **TruTV** - **Tube8** @@ -991,15 +930,11 @@ - **TV2DKBornholmPlay** - **TV4**: tv4.se and tv4play.se - **TV5MondePlus**: TV5MONDE+ - - **tv5unis** - - **tv5unis:video** - - **tv8.it** - **TVA** - **TVANouvelles** - **TVANouvellesArticle** - **TVC** - **TVCArticle** - - **TVer** - **tvigle**: Интернет-телевидение Tvigle.ru - **tvland.com** - **TVN24** @@ -1065,10 +1000,7 @@ - **Vidbit** - **Viddler** - **Videa** - - **video.arnes.si**: Arnes Video - **video.google:search**: Google Video search - - **video.sky.it** - - **video.sky.it:live** - **VideoDetective** - **videofy.me** - **videomore** @@ -1080,6 +1012,7 @@ - **vidme** - **vidme:user** - **vidme:user:likes** + - **Vidzi** - **vier**: vier.be and vijf.be - **vier:videos** - **viewlift** @@ -1109,7 +1042,7 @@ - **vk:wallpost** - **vlive** - **vlive:channel** - - **vlive:post** + - **vlive:playlist** - **Vodlocker** - **VODPl** - **VODPlatform** @@ -1124,12 +1057,10 @@ - **vrv** - **vrv:series** - **VShare** - - **VTM** - **VTXTV** - **vube**: Vube.com - **VuClip** - **VVVVID** - - **VVVVIDShow** - **VyboryMos** - **Vzaar** - **Wakanim** @@ -1152,7 +1083,6 @@ - **WeiboMobile** - **WeiqiTV**: WQTV - **Wistia** - - **WistiaPlaylist** - **wnl**: npo.nl, ntr.nl, omroepwnl.nl, zapp.nl and npo3.nl - **WorldStarHipHop** - **WSJ**: Wall Street Journal @@ -1160,7 +1090,7 @@ - **WWE** - **XBef** - **XboxClips** - - **XFileShare**: XFileShare based sites: Aparat, ClipWatching, GoUnlimited, GoVid, HolaVid, Streamty, TheVideoBee, Uqload, VidBom, vidlo, VidLocker, VidShare, VUp, WolfStream, XVideoSharing + - **XFileShare**: XFileShare based sites: ClipWatching, GoUnlimited, GoVid, HolaVid, Streamty, TheVideoBee, Uqload, VidBom, vidlo, VidLocker, VidShare, VUp, XVideoSharing - **XHamster** - **XHamsterEmbed** - **XHamsterUser** @@ -1184,8 +1114,6 @@ - **yahoo:japannews**: Yahoo! Japan News - **YandexDisk** - **yandexmusic:album**: Яндекс.Музыка - Альбом - - **yandexmusic:artist:albums**: Яндекс.Музыка - Артист - Альбомы - - **yandexmusic:artist:tracks**: Яндекс.Музыка - Артист - Треки - **yandexmusic:playlist**: Яндекс.Музыка - Плейлист - **yandexmusic:track**: Яндекс.Музыка - Трек - **YandexVideo** @@ -1203,24 +1131,25 @@ - **YourPorn** - **YourUpload** - **youtube**: YouTube.com + - **youtube:channel**: YouTube.com channels - **youtube:favorites**: YouTube.com favourite videos, ":ytfav" for short (requires authentication) - **youtube:history**: Youtube watch history, ":ythistory" for short (requires authentication) + - **youtube:live**: YouTube.com live streams - **youtube:playlist**: YouTube.com playlists + - **youtube:playlists**: YouTube.com user/channel playlists - **youtube:recommended**: YouTube.com recommended videos, ":ytrec" for short (requires authentication) - **youtube:search**: YouTube.com searches - **youtube:search:date**: YouTube.com searches, newest videos first + - **youtube:search_url**: YouTube.com search URLs + - **youtube:show**: YouTube.com (multi-season) shows - **youtube:subscriptions**: YouTube.com subscriptions feed, "ytsubs" keyword (requires authentication) - - **youtube:tab**: YouTube.com tab + - **youtube:user**: YouTube.com user videos (URL or "ytuser" keyword) - **youtube:watchlater**: Youtube watch later list, ":ytwatchlater" for short (requires authentication) - - **YoutubeYtBe** - - **YoutubeYtUser** - **Zapiks** + - **Zaq1** - **Zattoo** - **ZattooLive** - **ZDF** - **ZDFChannel** - - **Zhihu** - **zingmp3**: mp3.zing.vn - - **zingmp3:album** - - **zoom** - **Zype** diff --git a/test/helper.py b/test/helper.py index 6f2129eff..e62aab11e 100644 --- a/test/helper.py +++ b/test/helper.py @@ -1,24 +1,22 @@ from __future__ import unicode_literals import errno +import io import hashlib import json import os.path import re +import types import ssl import sys -import types -import unittest import youtube_dl.extractor from youtube_dl import YoutubeDL from youtube_dl.compat import ( - compat_open as open, compat_os_name, compat_str, ) from youtube_dl.utils import ( - IDENTITY, preferredencoding, write_string, ) @@ -29,10 +27,10 @@ def get_params(override=None): "parameters.json") LOCAL_PARAMETERS_FILE = os.path.join(os.path.dirname(os.path.abspath(__file__)), "local_parameters.json") - with open(PARAMETERS_FILE, encoding='utf-8') as pf: + with io.open(PARAMETERS_FILE, encoding='utf-8') as pf: parameters = json.load(pf) if os.path.exists(LOCAL_PARAMETERS_FILE): - with open(LOCAL_PARAMETERS_FILE, encoding='utf-8') as pf: + with io.open(LOCAL_PARAMETERS_FILE, encoding='utf-8') as pf: parameters.update(json.load(pf)) if override: parameters.update(override) @@ -74,8 +72,7 @@ class FakeYDL(YoutubeDL): def to_screen(self, s, skip_eol=None): print(s) - def trouble(self, *args, **kwargs): - s = args[0] if len(args) > 0 else kwargs.get('message', 'Missing message') + def trouble(self, s, tb=None): raise Exception(s) def download(self, x): @@ -92,17 +89,6 @@ class FakeYDL(YoutubeDL): self.report_warning = types.MethodType(report_warning, self) -class FakeLogger(object): - def debug(self, msg): - pass - - def warning(self, msg): - pass - - def error(self, msg): - pass - - def gettestcases(include_onlymatching=False): for ie in youtube_dl.extractor.gen_extractors(): for tc in ie.get_testcases(include_onlymatching): @@ -142,12 +128,6 @@ def expect_value(self, got, expected, field): self.assertTrue( contains_str in got, 'field %s (value: %r) should contain %r' % (field, got, contains_str)) - elif isinstance(expected, compat_str) and re.match(r'lambda \w+:', expected): - fn = eval(expected) - suite = expected.split(':', 1)[1].strip() - self.assertTrue( - fn(got), - 'Expected field %s to meet condition %s, but value %r failed ' % (field, suite, got)) elif isinstance(expected, type): self.assertTrue( isinstance(got, expected), @@ -157,7 +137,7 @@ def expect_value(self, got, expected, field): elif isinstance(expected, list) and isinstance(got, list): self.assertEqual( len(expected), len(got), - 'Expected a list of length %d, but got a list of length %d for field %s' % ( + 'Expect a list of length %d, but got a list of length %d for field %s' % ( len(expected), len(got), field)) for index, (item_got, item_expected) in enumerate(zip(got, expected)): type_got = type(item_got) @@ -181,18 +161,18 @@ def expect_value(self, got, expected, field): op, _, expected_num = expected.partition(':') expected_num = int(expected_num) if op == 'mincount': - assert_func = self.assertGreaterEqual + assert_func = assertGreaterEqual msg_tmpl = 'Expected %d items in field %s, but only got %d' elif op == 'maxcount': - assert_func = self.assertLessEqual + assert_func = assertLessEqual msg_tmpl = 'Expected maximum %d items in field %s, but got %d' elif op == 'count': - assert_func = self.assertEqual + assert_func = assertEqual msg_tmpl = 'Expected exactly %d items in field %s, but got %d' else: assert False assert_func( - len(got), expected_num, + self, len(got), expected_num, msg_tmpl % (expected_num, field, len(got))) return self.assertEqual( @@ -262,6 +242,27 @@ def assertRegexpMatches(self, text, regexp, msg=None): self.assertTrue(m, msg) +def assertGreaterEqual(self, got, expected, msg=None): + if not (got >= expected): + if msg is None: + msg = '%r not greater than or equal to %r' % (got, expected) + self.assertTrue(got >= expected, msg) + + +def assertLessEqual(self, got, expected, msg=None): + if not (got <= expected): + if msg is None: + msg = '%r not less than or equal to %r' % (got, expected) + self.assertTrue(got <= expected, msg) + + +def assertEqual(self, got, expected, msg=None): + if not (got == expected): + if msg is None: + msg = '%r not equal to %r' % (got, expected) + self.assertTrue(got == expected, msg) + + def expect_warnings(ydl, warnings_re): real_warning = ydl.report_warning @@ -279,7 +280,3 @@ def http_server_port(httpd): else: sock = httpd.socket return sock.getsockname()[1] - - -def expectedFailureIf(cond): - return unittest.expectedFailure if cond else IDENTITY diff --git a/test/parameters.json b/test/parameters.json index 864c9d130..7bf59c25f 100644 --- a/test/parameters.json +++ b/test/parameters.json @@ -18,6 +18,7 @@ "noprogress": false, "outtmpl": "%(id)s.%(ext)s", "password": null, + "playlistend": -1, "playliststart": 1, "prefer_free_formats": false, "quiet": false, @@ -36,7 +37,7 @@ "writeinfojson": true, "writesubtitles": false, "allsubtitles": false, - "listsubtitles": false, + "listssubtitles": false, "socket_timeout": 20, "fixup": "never" } diff --git a/test/test_InfoExtractor.py b/test/test_InfoExtractor.py index 09100a1d6..71f6608fe 100644 --- a/test/test_InfoExtractor.py +++ b/test/test_InfoExtractor.py @@ -3,36 +3,18 @@ from __future__ import unicode_literals # Allow direct execution +import io import os import sys import unittest - sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -import threading - -from test.helper import ( - expect_dict, - expect_value, - FakeYDL, - http_server_port, -) -from youtube_dl.compat import ( - compat_etree_fromstring, - compat_http_server, - compat_open as open, -) +from test.helper import FakeYDL, expect_dict, expect_value, http_server_port +from youtube_dl.compat import compat_etree_fromstring, compat_http_server from youtube_dl.extractor.common import InfoExtractor -from youtube_dl.extractor import ( - get_info_extractor, - YoutubeIE, -) -from youtube_dl.utils import ( - encode_data_uri, - ExtractorError, - RegexNotFoundError, - strip_jsonp, -) +from youtube_dl.extractor import YoutubeIE, get_info_extractor +from youtube_dl.utils import encode_data_uri, strip_jsonp, ExtractorError, RegexNotFoundError +import threading TEAPOT_RESPONSE_STATUS = 418 @@ -53,13 +35,13 @@ class InfoExtractorTestRequestHandler(compat_http_server.BaseHTTPRequestHandler) assert False -class DummyIE(InfoExtractor): +class TestIE(InfoExtractor): pass class TestInfoExtractor(unittest.TestCase): def setUp(self): - self.ie = DummyIE(FakeYDL()) + self.ie = TestIE(FakeYDL()) def test_ie_key(self): self.assertEqual(get_info_extractor(YoutubeIE.ie_key()), YoutubeIE) @@ -80,7 +62,6 @@ class TestInfoExtractor(unittest.TestCase): <meta name="og:test1" content='foo > < bar'/> <meta name="og:test2" content="foo >//< bar"/> <meta property=og-test3 content='Ill-formatted opengraph'/> - <meta property=og:test4 content=unquoted-value/> ''' self.assertEqual(ie._og_search_title(html), 'Foo') self.assertEqual(ie._og_search_description(html), 'Some video\'s description ') @@ -93,7 +74,6 @@ class TestInfoExtractor(unittest.TestCase): self.assertEqual(ie._og_search_property(('test0', 'test1'), html), 'foo > < bar') self.assertRaises(RegexNotFoundError, ie._og_search_property, 'test0', html, None, fatal=True) self.assertRaises(RegexNotFoundError, ie._og_search_property, ('test0', 'test00'), html, None, fatal=True) - self.assertEqual(ie._og_search_property('test4', html), 'unquoted-value') def test_html_search_meta(self): ie = self.ie @@ -118,123 +98,6 @@ class TestInfoExtractor(unittest.TestCase): self.assertRaises(RegexNotFoundError, ie._html_search_meta, 'z', html, None, fatal=True) self.assertRaises(RegexNotFoundError, ie._html_search_meta, ('z', 'x'), html, None, fatal=True) - def test_search_nextjs_data(self): - html = ''' -<!DOCTYPE html> -<html> -<head> - <meta http-equiv="content-type" content= - "text/html; charset=utf-8"> - <meta name="viewport" content="width=device-width"> - <title>Test _search_nextjs_data() - - -
-
-
-
-
-
-
-
-
-
-
- -
-
-
- - - -''' - search = self.ie._search_nextjs_data(html, 'testID') - self.assertEqual(search['props']['pageProps']['video']['id'], 'testid') - search = self.ie._search_nextjs_data( - 'no next.js data here, move along', 'testID', default={'status': 0}) - self.assertEqual(search['status'], 0) - - def test_search_nuxt_data(self): - html = ''' - - - - - Nuxt.js Test Page - - - - -
-

Example heading

-
-

Decoy text

-
-
- - - - -''' - search = self.ie._search_nuxt_data(html, 'testID') - self.assertEqual(search['track']['id'], 'testid') - - def test_search_json_ld_realworld(self): - # https://github.com/ytdl-org/youtube-dl/issues/23306 - expect_dict( - self, - self.ie._search_json_ld(r'''''', None), - { - 'title': '1 On 1 With Kleio', - 'description': 'Kleio Valentien', - 'url': 'https://gvideo.eporner.com/xN49A1cT3eB/xN49A1cT3eB.mp4', - 'timestamp': 1449347075, - 'duration': 743.0, - 'view_count': 1120958, - 'width': 1920, - 'height': 1080, - }) - def test_download_json(self): uri = encode_data_uri(b'{"foo": "blah"}', 'application/json') self.assertEqual(self.ie._download_json(uri, None), {'foo': 'blah'}) @@ -245,18 +108,6 @@ class TestInfoExtractor(unittest.TestCase): self.assertEqual(self.ie._download_json(uri, None, fatal=False), None) def test_parse_html5_media_entries(self): - # inline video tag - expect_dict( - self, - self.ie._parse_html5_media_entries( - 'https://127.0.0.1/video.html', - r'