Inject.py now tries to detect encoding before parsing HTML with BeautifulSoup

CONTRIBUTORS.md

@@ -11,6 +11,7 @@
 - @mmetince
 - @niallmerrigan
 - @auraltension
+- @HAMIDx9
 
 #Unintentional contributors and/or projects that I stole code from
 

plugins/inject.py

@@ -57,27 +57,37 @@ class Inject(Plugin):
     
     def response(self, response, request, data):
 
+        encoding = None
         ip = response.getClientIP()
         hn = response.getRequestHostname()
-        mime = response.headers['Content-Type']
+
+        try:
+            mime = response.headers['Content-Type']
+        except KeyError:
+            return
+
+        if "charset" in mime:
+            match = re.search('charset=(.*)', mime)
+            if match:
+                encoding = match.group(1).strip().replace('"', "")
+            else:
+                try:
+                    encoding = chardet.detect(data)["encoding"]
+                except:
+                    pass
+        else:
+            try:
+                encoding = chardet.detect(data)["encoding"]
+            except:
+                pass
 
         if self._should_inject(ip, hn) and self._ip_filter(ip) and self._host_filter(hn) and (hn not in self.ip) and ("text/html" in mime):
 
-	    if "charset" in mime:
+    	    if encoding is not None:
-		match = re.search('charset=(.*)', mime)
+                html = BeautifulSoup(data.decode(encoding, "ignore"), "lxml")
-		if match:
+    	    else:
-			encoding = match.group(1).strip().replace('"', "")
+                html = BeautifulSoup(data, "lxml")
-	    else:
-	    	try:
-			encoding = chardet.detect(data)["encoding"]
-		except:
-			encoding = None
 
-	    if encoding:		
-	            html = BeautifulSoup(data.decode(encoding, "ignore"), "lxml")
-	    else:
-	            html = BeautifulSoup(data, "lxml") # let bs find the encoding
-		
             if html.body:
 
                 if self.html_url:

requirements.txt

@@ -1,7 +1,7 @@
 git+git://github.com/kti/python-netfilterqueue
 pyinotify
-pycrypto>=2.6
+pycrypto
-pyasn1>=0.1.7
+pyasn1
 cryptography
 Pillow
 netaddr
@@ -23,4 +23,4 @@ python-magic
 msgpack-python
 requests
 pypcap
-chardet
+chardet