new: the events.stream will now parse and display properly interesting http requests and responses

modules/net_sniff_http.go

@@ -3,64 +3,145 @@ package modules
 import (
 	"bufio"
 	"bytes"
+	"compress/gzip"
 	"io/ioutil"
 	"net/http"
-	"net/url"
+	"strings"
 
 	"github.com/bettercap/bettercap/core"
 
 	"github.com/google/gopacket"
 	"github.com/google/gopacket/layers"
+
+	"github.com/dustin/go-humanize"
 )
 
 type HTTPRequest struct {
-	Method  string      `json:"method"`
-	Host    string      `json:"host"`
-	URL     string      `json:"url:"`
-	Headers http.Header `json:"headers"`
-	Form    url.Values  `json:"form"`
-	Body    []byte      `json:"body"`
+	Method      string      `json:"method"`
+	Proto       string      `json:"proto"`
+	Host        string      `json:"host"`
+	URL         string      `json:"url:"`
+	Headers     http.Header `json:"headers"`
+	ContentType string      `json:"content_type"`
+	Body        []byte      `json:"body"`
+}
+
+type HTTPResponse struct {
+	Protocol         string      `json:"protocol"`
+	Status           string      `json:"status"`
+	StatusCode       int         `json:"status_code"`
+	Headers          http.Header `json:"headers"`
+	Body             []byte      `json:"body"`
+	ContentLength    int64       `json:"content_length"`
+	ContentType      string      `json:"content_type"`
+	TransferEncoding []string    `json:"transfer_encoding"`
 }
 
 func toSerializableRequest(req *http.Request) HTTPRequest {
 	body := []byte(nil)
-	form := (url.Values)(nil)
-
-	if err := req.ParseForm(); err == nil {
-		form = req.Form
-	} else if req.Body != nil {
+	ctype := "?"
+	if req.Body != nil {
 		body, _ = ioutil.ReadAll(req.Body)
 	}
 
+	for name, values := range req.Header {
+		if strings.ToLower(name) == "content-type" {
+			for _, value := range values {
+				ctype = value
+			}
+		}
+	}
+
 	return HTTPRequest{
-		Method:  req.Method,
-		Host:    req.Host,
-		URL:     req.URL.String(),
-		Headers: req.Header,
-		Form:    form,
-		Body:    body,
+		Method:      req.Method,
+		Proto:       req.Proto,
+		Host:        req.Host,
+		URL:         req.URL.String(),
+		Headers:     req.Header,
+		ContentType: ctype,
+		Body:        body,
+	}
+}
+
+func toSerializableResponse(res *http.Response) HTTPResponse {
+	body := []byte(nil)
+	ctype := "?"
+	cenc := ""
+	for name, values := range res.Header {
+		name = strings.ToLower(name)
+		if name == "content-type" {
+			for _, value := range values {
+				ctype = value
+			}
+		} else if name == "content-encoding" {
+			for _, value := range values {
+				cenc = value
+			}
+		}
+	}
+
+	if res.Body != nil {
+		body, _ = ioutil.ReadAll(res.Body)
+	}
+
+	// attempt decompression, but since this has been parsed by just
+	// a tcp packet, it will probably fail
+	if body != nil && strings.Contains(cenc, "gzip") {
+		buffer := bytes.NewBuffer(body)
+		uncompressed := bytes.Buffer{}
+		if reader, err := gzip.NewReader(buffer); err == nil {
+			if _, err = uncompressed.ReadFrom(reader); err == nil {
+				body = uncompressed.Bytes()
+			}
+		}
+	}
+
+	return HTTPResponse{
+		Protocol:         res.Proto,
+		Status:           res.Status,
+		StatusCode:       res.StatusCode,
+		Headers:          res.Header,
+		Body:             body,
+		ContentLength:    res.ContentLength,
+		ContentType:      ctype,
+		TransferEncoding: res.TransferEncoding,
 	}
 }
 
 func httpParser(ip *layers.IPv4, pkt gopacket.Packet, tcp *layers.TCP) bool {
 	data := tcp.Payload
-	reader := bufio.NewReader(bytes.NewReader(data))
-	req, err := http.ReadRequest(reader)
-
-	if err == nil {
+	if req, err := http.ReadRequest(bufio.NewReader(bytes.NewReader(data))); err == nil {
 		NewSnifferEvent(
 			pkt.Metadata().Timestamp,
-			"http",
+			"http.request",
 			ip.SrcIP.String(),
 			req.Host,
 			toSerializableRequest(req),
-			"%s %s %s %s%s %s",
+			"%s %s %s %s%s",
 			core.W(core.BG_RED+core.FG_BLACK, "http"),
 			vIP(ip.SrcIP),
 			core.W(core.BG_LBLUE+core.FG_BLACK, req.Method),
 			core.Yellow(req.Host),
 			vURL(req.URL.String()),
-			core.Dim(req.UserAgent()),
+		).Push()
+
+		return true
+	} else if res, err := http.ReadResponse(bufio.NewReader(bytes.NewReader(data)), nil); err == nil {
+		sres := toSerializableResponse(res)
+		NewSnifferEvent(
+			pkt.Metadata().Timestamp,
+			"http.response",
+			ip.SrcIP.String(),
+			ip.DstIP.String(),
+			sres,
+			"%s %s:%d %s -> %s (%s %s)",
+			core.W(core.BG_RED+core.FG_BLACK, "http"),
+			vIP(ip.SrcIP),
+			tcp.SrcPort,
+			core.Bold(res.Status),
+			vIP(ip.DstIP),
+			core.Dim(humanize.Bytes(uint64(len(sres.Body)))),
+			core.Yellow(sres.ContentType),
 		).Push()
 
 		return true