diff --git a/.gitignore b/.gitignore index f07fc5d60..564bde1d1 100644 --- a/.gitignore +++ b/.gitignore @@ -16,3 +16,4 @@ youtube-dl.exe youtube-dl.tar.gz .coverage cover/ +updates_key.pem diff --git a/Makefile b/Makefile index 818d93bdb..b49af6a7a 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ youtube-dl.bash-completion: youtube_dl/*.py devscripts/bash-completion.in python devscripts/bash-completion.py youtube-dl.tar.gz: all - tar -czf youtube-dl.tar.gz -s "|^./|./youtube-dl/|" \ + tar -czf youtube-dl.tar.gz -s "|^./|./youtube-dl/|" --exclude="updates_key.pem" \ --exclude="*.pyc" --exclude="*.pyo" --exclude="*~" --exclude="youtube-dl.exe" \ --exclude="wine-py2exe/" --exclude="py2exe.log" --exclude="*.kate-swp" \ --exclude="build/" --exclude="dist/" --exclude="MANIFEST" --exclude=".git/" . diff --git a/README.md b/README.md index 407b27970..993aef49e 100644 --- a/README.md +++ b/README.md @@ -196,7 +196,7 @@ Bugs and suggestions should be reported at: Copyright © 2006-)(?P[0-9]{4})', u'Copyright © 2006-' + year, content) + if content != newc: + tmpFn = fn + '.part' + with io.open(tmpFn, 'wt', encoding='utf-8') as outf: + outf.write(newc) + os.rename(tmpFn, fn) diff --git a/devscripts/transition_helper_exe/youtube-dl.py b/devscripts/transition_helper_exe/youtube-dl.py index 409f980bc..dbb4c99e1 100644 --- a/devscripts/transition_helper_exe/youtube-dl.py +++ b/devscripts/transition_helper_exe/youtube-dl.py @@ -2,17 +2,48 @@ import sys, os import urllib2 +import json, hashlib + +def rsa_verify(message, signature, key): + from struct import pack + from hashlib import sha256 + from sys import version_info + def b(x): + if version_info[0] == 2: return x + else: return x.encode('latin1') + assert(type(message) == type(b(''))) + block_size = 0 + n = key[0] + while n: + block_size += 1 + n >>= 8 + signature = pow(int(signature, 16), key[1], key[0]) + raw_bytes = [] + while signature: + raw_bytes.insert(0, pack("B", signature & 0xFF)) + signature >>= 8 + signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes) + if signature[0:2] != b('\x00\x01'): return False + signature = signature[2:] + if not b('\x00') in signature: return False + signature = signature[signature.index(b('\x00'))+1:] + if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False + signature = signature[19:] + if signature != sha256(message).digest(): return False + return True sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n') sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n') -sys.stderr.write(u'The new location of the binaries is https://github.com/rg3/youtube-dl/downloads, not the git repository.\n\n') +sys.stderr.write(u'From now on, get the binaries from http://rg3.github.com/youtube-dl/download.html, not from the git repository.\n\n') raw_input() filename = sys.argv[0] -API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads" -EXE_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl.exe" +UPDATE_URL = "http://rg3.github.com/youtube-dl/update/" +VERSION_URL = UPDATE_URL + 'LATEST_VERSION' +JSON_URL = UPDATE_URL + 'versions.json' +UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537) if not os.access(filename, os.W_OK): sys.exit('ERROR: no write permissions on %s' % filename) @@ -23,13 +54,35 @@ if not os.access(directory, os.W_OK): sys.exit('ERROR: no write permissions on %s' % directory) try: - urlh = urllib2.urlopen(EXE_URL) + versions_info = urllib2.urlopen(JSON_URL).read().decode('utf-8') + versions_info = json.loads(versions_info) +except: + sys.exit(u'ERROR: can\'t obtain versions info. Please try again later.') +if not 'signature' in versions_info: + sys.exit(u'ERROR: the versions file is not signed or corrupted. Aborting.') +signature = versions_info['signature'] +del versions_info['signature'] +if not rsa_verify(json.dumps(versions_info, sort_keys=True), signature, UPDATES_RSA_KEY): + sys.exit(u'ERROR: the versions file signature is invalid. Aborting.') + +version = versions_info['versions'][versions_info['latest']] + +try: + urlh = urllib2.urlopen(version['exe'][0]) newcontent = urlh.read() urlh.close() +except (IOError, OSError) as err: + sys.exit('ERROR: unable to download latest version') + +newcontent_hash = hashlib.sha256(newcontent).hexdigest() +if newcontent_hash != version['exe'][1]: + sys.exit(u'ERROR: the downloaded file hash does not match. Aborting.') + +try: with open(exe + '.new', 'wb') as outf: outf.write(newcontent) except (IOError, OSError) as err: - sys.exit('ERROR: unable to download latest version') + sys.exit(u'ERROR: unable to write the new version') try: bat = os.path.join(directory, 'youtube-dl-updater.bat') diff --git a/youtube-dl b/youtube-dl index d5ca2d4ba..e6f05c173 100755 --- a/youtube-dl +++ b/youtube-dl @@ -1,15 +1,44 @@ #!/usr/bin/env python import sys, os +import json, hashlib try: import urllib.request as compat_urllib_request except ImportError: # Python 2 import urllib2 as compat_urllib_request +def rsa_verify(message, signature, key): + from struct import pack + from hashlib import sha256 + from sys import version_info + def b(x): + if version_info[0] == 2: return x + else: return x.encode('latin1') + assert(type(message) == type(b(''))) + block_size = 0 + n = key[0] + while n: + block_size += 1 + n >>= 8 + signature = pow(int(signature, 16), key[1], key[0]) + raw_bytes = [] + while signature: + raw_bytes.insert(0, pack("B", signature & 0xFF)) + signature >>= 8 + signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes) + if signature[0:2] != b('\x00\x01'): return False + signature = signature[2:] + if not b('\x00') in signature: return False + signature = signature[signature.index(b('\x00'))+1:] + if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False + signature = signature[19:] + if signature != sha256(message).digest(): return False + return True + sys.stderr.write(u'Hi! We changed distribution method and now youtube-dl needs to update itself one more time.\n') sys.stderr.write(u'This will only happen once. Simply press enter to go on. Sorry for the trouble!\n') -sys.stderr.write(u'The new location of the binaries is https://github.com/rg3/youtube-dl/downloads, not the git repository.\n\n') +sys.stderr.write(u'From now on, get the binaries from http://rg3.github.com/youtube-dl/download.html, not from the git repository.\n\n') try: raw_input() @@ -18,19 +47,39 @@ except NameError: # Python 3 filename = sys.argv[0] -API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads" -BIN_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl" +UPDATE_URL = "http://rg3.github.com/youtube-dl/update/" +VERSION_URL = UPDATE_URL + 'LATEST_VERSION' +JSON_URL = UPDATE_URL + 'versions.json' +UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537) if not os.access(filename, os.W_OK): sys.exit('ERROR: no write permissions on %s' % filename) try: - urlh = compat_urllib_request.urlopen(BIN_URL) + versions_info = compat_urllib_request.urlopen(JSON_URL).read().decode('utf-8') + versions_info = json.loads(versions_info) +except: + sys.exit(u'ERROR: can\'t obtain versions info. Please try again later.') +if not 'signature' in versions_info: + sys.exit(u'ERROR: the versions file is not signed or corrupted. Aborting.') +signature = versions_info['signature'] +del versions_info['signature'] +if not rsa_verify(json.dumps(versions_info, sort_keys=True).encode('utf-8'), signature, UPDATES_RSA_KEY): + sys.exit(u'ERROR: the versions file signature is invalid. Aborting.') + +version = versions_info['versions'][versions_info['latest']] + +try: + urlh = compat_urllib_request.urlopen(version['bin'][0]) newcontent = urlh.read() urlh.close() except (IOError, OSError) as err: sys.exit('ERROR: unable to download latest version') +newcontent_hash = hashlib.sha256(newcontent).hexdigest() +if newcontent_hash != version['bin'][1]: + sys.exit(u'ERROR: the downloaded file hash does not match. Aborting.') + try: with open(filename, 'wb') as outf: outf.write(newcontent) diff --git a/youtube-dl.exe b/youtube-dl.exe index a11878986..45eee04bb 100644 Binary files a/youtube-dl.exe and b/youtube-dl.exe differ diff --git a/youtube_dl/InfoExtractors.py b/youtube_dl/InfoExtractors.py index e3cf7e115..d74751a55 100755 --- a/youtube_dl/InfoExtractors.py +++ b/youtube_dl/InfoExtractors.py @@ -1093,7 +1093,6 @@ class ArteTvIE(InfoExtractor): self._downloader.to_screen(u'[arte.tv] %s: Extracting information' % video_id) def fetch_webpage(self, url): - self._downloader.increment_downloads() request = compat_urllib_request.Request(url) try: self.report_download_webpage(url) diff --git a/youtube_dl/__init__.py b/youtube_dl/__init__.py index d12ece21e..8068810ca 100644 --- a/youtube_dl/__init__.py +++ b/youtube_dl/__init__.py @@ -34,106 +34,15 @@ import socket import subprocess import sys import warnings +import platform from .utils import * +from .update import update_self from .version import __version__ from .FileDownloader import * from .InfoExtractors import * from .PostProcessor import * -def updateSelf(downloader, filename): - """Update the program file with the latest version from the repository""" - - # TODO: at least, check https certificates - - from zipimport import zipimporter - - API_URL = "https://api.github.com/repos/rg3/youtube-dl/downloads" - BIN_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl" - EXE_URL = "https://github.com/downloads/rg3/youtube-dl/youtube-dl.exe" - - if hasattr(sys, "frozen"): # PY2EXE - if not os.access(filename, os.W_OK): - sys.exit('ERROR: no write permissions on %s' % filename) - - downloader.to_screen(u'Updating to latest version...') - - urla = compat_urllib_request.urlopen(API_URL) - download = filter(lambda x: x["name"] == "youtube-dl.exe", json.loads(urla.read())) - if not download: - downloader.to_screen(u'ERROR: can\'t find the current version. Please try again later.') - return - newversion = download[0]["description"].strip() - if newversion == __version__: - downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')') - return - urla.close() - - exe = os.path.abspath(filename) - directory = os.path.dirname(exe) - if not os.access(directory, os.W_OK): - sys.exit('ERROR: no write permissions on %s' % directory) - - try: - urlh = compat_urllib_request.urlopen(EXE_URL) - newcontent = urlh.read() - urlh.close() - with open(exe + '.new', 'wb') as outf: - outf.write(newcontent) - except (IOError, OSError) as err: - sys.exit('ERROR: unable to download latest version') - - try: - bat = os.path.join(directory, 'youtube-dl-updater.bat') - b = open(bat, 'w') - b.write(""" -echo Updating youtube-dl... -ping 127.0.0.1 -n 5 -w 1000 > NUL -move /Y "%s.new" "%s" -del "%s" - \n""" %(exe, exe, bat)) - b.close() - - os.startfile(bat) - except (IOError, OSError) as err: - sys.exit('ERROR: unable to overwrite current version') - - elif isinstance(globals().get('__loader__'), zipimporter): # UNIX ZIP - if not os.access(filename, os.W_OK): - sys.exit('ERROR: no write permissions on %s' % filename) - - downloader.to_screen(u'Updating to latest version...') - - urla = compat_urllib_request.urlopen(API_URL) - download = [x for x in json.loads(urla.read().decode('utf8')) if x["name"] == "youtube-dl"] - if not download: - downloader.to_screen(u'ERROR: can\'t find the current version. Please try again later.') - return - newversion = download[0]["description"].strip() - if newversion == __version__: - downloader.to_screen(u'youtube-dl is up-to-date (' + __version__ + ')') - return - urla.close() - - try: - urlh = compat_urllib_request.urlopen(BIN_URL) - newcontent = urlh.read() - urlh.close() - except (IOError, OSError) as err: - sys.exit('ERROR: unable to download latest version') - - try: - with open(filename, 'wb') as outf: - outf.write(newcontent) - except (IOError, OSError) as err: - sys.exit('ERROR: unable to overwrite current version') - - else: - downloader.to_screen(u'It looks like you installed youtube-dl with pip or setup.py. Please use that to update.') - return - - downloader.to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.') - def parseOpts(): def _readOptions(filename_bytes): try: @@ -567,6 +476,17 @@ def _real_main(): }) if opts.verbose: + fd.to_screen(u'[debug] youtube-dl version ' + __version__) + try: + sp = subprocess.Popen(['git', 'rev-parse', '--short', 'HEAD'], stdout=subprocess.PIPE, stderr=subprocess.PIPE, + cwd=os.path.dirname(os.path.abspath(__file__))) + out, err = sp.communicate() + out = out.decode().strip() + if re.match('[0-9a-f]+', out): + fd.to_screen(u'[debug] Git HEAD: ' + out) + except: + pass + fd.to_screen(u'[debug] Python version %s - %s' %(platform.python_version(), platform.platform())) fd.to_screen(u'[debug] Proxy map: ' + str(proxy_handler.proxies)) for extractor in extractors: @@ -578,7 +498,7 @@ def _real_main(): # Update version if opts.update_self: - updateSelf(fd, sys.argv[0]) + update_self(fd.to_screen, opts.verbose, sys.argv[0]) # Maybe do nothing if len(all_urls) < 1: diff --git a/youtube_dl/update.py b/youtube_dl/update.py new file mode 100644 index 000000000..f6e3e5c69 --- /dev/null +++ b/youtube_dl/update.py @@ -0,0 +1,160 @@ +import json +import traceback +import hashlib +from zipimport import zipimporter + +from .utils import * +from .version import __version__ + +def rsa_verify(message, signature, key): + from struct import pack + from hashlib import sha256 + from sys import version_info + def b(x): + if version_info[0] == 2: return x + else: return x.encode('latin1') + assert(type(message) == type(b(''))) + block_size = 0 + n = key[0] + while n: + block_size += 1 + n >>= 8 + signature = pow(int(signature, 16), key[1], key[0]) + raw_bytes = [] + while signature: + raw_bytes.insert(0, pack("B", signature & 0xFF)) + signature >>= 8 + signature = (block_size - len(raw_bytes)) * b('\x00') + b('').join(raw_bytes) + if signature[0:2] != b('\x00\x01'): return False + signature = signature[2:] + if not b('\x00') in signature: return False + signature = signature[signature.index(b('\x00'))+1:] + if not signature.startswith(b('\x30\x31\x30\x0D\x06\x09\x60\x86\x48\x01\x65\x03\x04\x02\x01\x05\x00\x04\x20')): return False + signature = signature[19:] + if signature != sha256(message).digest(): return False + return True + +def update_self(to_screen, verbose, filename): + """Update the program file with the latest version from the repository""" + + UPDATE_URL = "http://rg3.github.com/youtube-dl/update/" + VERSION_URL = UPDATE_URL + 'LATEST_VERSION' + JSON_URL = UPDATE_URL + 'versions.json' + UPDATES_RSA_KEY = (0x9d60ee4d8f805312fdb15a62f87b95bd66177b91df176765d13514a0f1754bcd2057295c5b6f1d35daa6742c3ffc9a82d3e118861c207995a8031e151d863c9927e304576bc80692bc8e094896fcf11b66f3e29e04e3a71e9a11558558acea1840aec37fc396fb6b65dc81a1c4144e03bd1c011de62e3f1357b327d08426fe93, 65537) + + + if not isinstance(globals().get('__loader__'), zipimporter) and not hasattr(sys, "frozen"): + to_screen(u'It looks like you installed youtube-dl with pip, setup.py or a tarball. Please use that to update.') + return + + # Check if there is a new version + try: + newversion = compat_urllib_request.urlopen(VERSION_URL).read().decode('utf-8').strip() + except: + if verbose: to_screen(compat_str(traceback.format_exc())) + to_screen(u'ERROR: can\'t find the current version. Please try again later.') + return + if newversion == __version__: + to_screen(u'youtube-dl is up-to-date (' + __version__ + ')') + return + + # Download and check versions info + try: + versions_info = compat_urllib_request.urlopen(JSON_URL).read().decode('utf-8') + versions_info = json.loads(versions_info) + except: + if verbose: to_screen(compat_str(traceback.format_exc())) + to_screen(u'ERROR: can\'t obtain versions info. Please try again later.') + return + if not 'signature' in versions_info: + to_screen(u'ERROR: the versions file is not signed or corrupted. Aborting.') + return + signature = versions_info['signature'] + del versions_info['signature'] + if not rsa_verify(json.dumps(versions_info, sort_keys=True).encode('utf-8'), signature, UPDATES_RSA_KEY): + to_screen(u'ERROR: the versions file signature is invalid. Aborting.') + return + + to_screen(u'Updating to version ' + versions_info['latest'] + '...') + version = versions_info['versions'][versions_info['latest']] + if version.get('notes'): + to_screen(u'PLEASE NOTE:') + for note in version['notes']: + to_screen(note) + + if not os.access(filename, os.W_OK): + to_screen(u'ERROR: no write permissions on %s' % filename) + return + + # Py2EXE + if hasattr(sys, "frozen"): + exe = os.path.abspath(filename) + directory = os.path.dirname(exe) + if not os.access(directory, os.W_OK): + to_screen(u'ERROR: no write permissions on %s' % directory) + return + + try: + urlh = compat_urllib_request.urlopen(version['exe'][0]) + newcontent = urlh.read() + urlh.close() + except (IOError, OSError) as err: + if verbose: to_screen(compat_str(traceback.format_exc())) + to_screen(u'ERROR: unable to download latest version') + return + + newcontent_hash = hashlib.sha256(newcontent).hexdigest() + if newcontent_hash != version['exe'][1]: + to_screen(u'ERROR: the downloaded file hash does not match. Aborting.') + return + + try: + with open(exe + '.new', 'wb') as outf: + outf.write(newcontent) + except (IOError, OSError) as err: + if verbose: to_screen(compat_str(traceback.format_exc())) + to_screen(u'ERROR: unable to write the new version') + return + + try: + bat = os.path.join(directory, 'youtube-dl-updater.bat') + b = open(bat, 'w') + b.write(""" +echo Updating youtube-dl... +ping 127.0.0.1 -n 5 -w 1000 > NUL +move /Y "%s.new" "%s" +del "%s" + \n""" %(exe, exe, bat)) + b.close() + + os.startfile(bat) + except (IOError, OSError) as err: + if verbose: to_screen(compat_str(traceback.format_exc())) + to_screen(u'ERROR: unable to overwrite current version') + return + + # Zip unix package + elif isinstance(globals().get('__loader__'), zipimporter): + try: + urlh = compat_urllib_request.urlopen(version['bin'][0]) + newcontent = urlh.read() + urlh.close() + except (IOError, OSError) as err: + if verbose: to_screen(compat_str(traceback.format_exc())) + to_screen(u'ERROR: unable to download latest version') + return + + newcontent_hash = hashlib.sha256(newcontent).hexdigest() + if newcontent_hash != version['bin'][1]: + to_screen(u'ERROR: the downloaded file hash does not match. Aborting.') + return + + try: + with open(filename, 'wb') as outf: + outf.write(newcontent) + except (IOError, OSError) as err: + if verbose: to_screen(compat_str(traceback.format_exc())) + to_screen(u'ERROR: unable to overwrite current version') + return + + to_screen(u'Updated youtube-dl. Restart youtube-dl to use the new version.') diff --git a/youtube_dl/utils.py b/youtube_dl/utils.py index 463804e18..e9336bcdd 100644 --- a/youtube_dl/utils.py +++ b/youtube_dl/utils.py @@ -154,6 +154,7 @@ std_headers = { 'Accept-Encoding': 'gzip, deflate', 'Accept-Language': 'en-us,en;q=0.5', } + def preferredencoding(): """Get preferred encoding. @@ -187,7 +188,6 @@ else: with open(fn, 'w', encoding='utf-8') as f: json.dump(obj, f) - def htmlentity_transform(matchobj): """Transforms an HTML entity to a character.